How to invoke a custom operation in Views

Those are "operations". That view field is controlled by the EntityOperations field.

It's pretty easy to add a new operation for an entity by implementing hook_entity_operation.

use Drupal\Core\Entity\EntityInterface;
use Drupal\Core\Url;

/**
 * Implements hook_entity_operation().
 */
function mymodule_entity_operation(\Drupal\Core\Entity\EntityInterface $entity) {
  $operations = [];
  if ($entity->getEntityTypeId() == 'node' && someOtherCondition($entity)) {
    $operations['flag_as_spam'] = [
      'title' => t('Flag as Spam'),
      // Replace with the route you've created to set the node value.
      'url' => Url::fromRoute('entity.node.canonical', ['node' => $entity->id()]),
      'weight' => 50,
    ];
  }
  return $operations;
}