How to deal with tuning error with TF model: InvalidArgumentError: Cannot convert a Tensor of dtype variant to a NumPy array

[wjlgatech]

I tried to do hyperparameter tuning for a TF model but get into an error: InvalidArgumentError: Cannot convert a Tensor of dtype variant to a NumPy array

In following code create_model() can be trained and make predictions without any error. However, when I put it under flaml.tune.run(), error occured.

import ray
from ray import tune
import flaml
import time

# define search space
max_num_epoch = 100
config={"rating_weight":tune.uniform(lower=0, upper=1),
        "n_layers": tune.choice([1,2,3]), 
        "n_units": tune.choice([16, 32, 64, 128, 256]), 
        "lr": tune.loguniform(lower=1e-4, upper=1e-1),
        "epochs": tune.loguniform(lower=1, upper=max_num_epoch),
        "loss_name": tune.choice(['listwise', 'mse', 'pairwise']),
        "flag_use_norm": tune.choice([True, False])
        }

# define evalute_config() with search space
def evaluate_config(config:dict, return_model=False):
    """evaluate the performance of a DeepCrossMultitaskModel model using config"""

    # assign number of units for each layer
    layer_sizes = []
    for i in range(config['n_layers'] -1):
        layer_sizes.append(config['n_units'])
    
    model = create_model(#embedding_dimension=config['embedding_dimension'],
                        flag_use_norm=config['flag_use_norm'],
                        rating_weight=config['rating_weight'], 
                        retrieval_weight=1-config['rating_weight'], 
                        layer_sizes=layer_sizes,
                        #user_layer_sizes=user_layer_sizes, 
                        #item_layer_sizes=item_layer_sizes,
                        loss_name=config['loss_name'],
                        lr=config['lr']
                        )
    history = model.fit(
                        cached_train,
                        validation_data=cached_test,
                        validation_freq=5,
                        epochs=config['epochs'],
                        verbose=0)

    accuracy = history.history['factorized_top_k/top_100_categorical_accuracy']
    print(f"------Train data: Top-100 accuracy = {accuracy}.")

    # evaluate the model on test data
    metrics = model.evaluate(cached_test, return_dict=True)

    print(f"======Test data: metrics={metrics}.======")
    if return_model:
        return model, metrics
    tune.report(top_100_accu=metrics['factorized_top_k/top_100_categorical_accuracy'],
                ndcg_metric=metrics['ndcg_metric'],
                rmse=metrics["RMSE"]
    )

np.random.seed(123)

# launch tuning
start_time = time.time()
result = flaml.tune.run(
    evaluate_config,
    search_alg=flaml.CFO(
        space=config,
        metric='ndcg_metric',
        mode='max',
        low_cost_partial_config={"num_train_epochs": 1}),
    resources_per_trial={"gpu": 4, "cpu": 4},
    local_dir='logs/',
    num_samples=-1, # number of trials, -1 means unlimited
    time_budget_s=3600,
    use_ray=True,
    )

The detailed error message is:

---------------------------------------------------------------------------
InvalidArgumentError                      Traceback (most recent call last)
/tmp/ipykernel_3035220/80379931.py in <module>
     59 # launch tuning
     60 start_time = time.time()
---> 61 result = flaml.tune.run(
     62     evaluate_config,
     63     search_alg=flaml.CFO(

/napi/k_working_dir/ds_test/shared-space/venv/lib/python3.9/site-packages/flaml/tune/tune.py in run(evaluation_function, config, low_cost_partial_config, cat_hp_cost, metric, mode, time_budget_s, points_to_evaluate, evaluated_rewards, resource_attr, min_resource, max_resource, reduction_factor, scheduler, search_alg, verbose, local_dir, num_samples, resources_per_trial, config_constraints, metric_constraints, max_failure, use_ray, use_incumbent_result_in_evaluation)
    404             )
    405         _use_ray = True
--> 406         return tune.run(
    407             evaluation_function,
    408             metric=metric,

/napi/k_working_dir/ds_test/shared-space/venv/lib/python3.9/site-packages/ray/tune/tune.py in run(run_or_experiment, name, metric, mode, stop, time_budget_s, config, resources_per_trial, num_samples, local_dir, search_alg, scheduler, keep_checkpoints_num, checkpoint_score_attr, checkpoint_freq, checkpoint_at_end, verbose, progress_reporter, log_to_file, trial_name_creator, trial_dirname_creator, sync_config, export_formats, max_failures, fail_fast, restore, server_port, resume, reuse_actors, trial_executor, raise_on_failed_trial, callbacks, max_concurrent_trials, queue_trials, loggers, _remote)
    423     for i, exp in enumerate(experiments):
    424         if not isinstance(exp, Experiment):
--> 425             experiments[i] = Experiment(
    426                 name=name,
    427                 run=exp,

/napi/k_working_dir/ds_test/shared-space/venv/lib/python3.9/site-packages/ray/tune/experiment.py in __init__(self, name, run, stop, time_budget_s, config, resources_per_trial, num_samples, local_dir, sync_config, trial_name_creator, trial_dirname_creator, log_to_file, checkpoint_freq, checkpoint_at_end, keep_checkpoints_num, checkpoint_score_attr, export_formats, max_failures, restore)
    111                     "checkpointable function. You can specify checkpoints "
    112                     "within your trainable function.")
--> 113         self._run_identifier = Experiment.register_if_needed(run)
    114         self.name = name or self._run_identifier
    115 

/napi/k_working_dir/ds_test/shared-space/venv/lib/python3.9/site-packages/ray/tune/experiment.py in register_if_needed(cls, run_object)
    256                     "No name detected on trainable. Using {}.".format(name))
    257             try:
--> 258                 register_trainable(name, run_object)
    259             except (TypeError, PicklingError) as e:
    260                 extra_msg = ("Other options: "

/napi/k_working_dir/ds_test/shared-space/venv/lib/python3.9/site-packages/ray/tune/registry.py in register_trainable(name, trainable, warn)
     74         raise TypeError("Second argument must be convertable to Trainable",
     75                         trainable)
---> 76     _global_registry.register(TRAINABLE_CLASS, name, trainable)
     77 
     78 

/napi/k_working_dir/ds_test/shared-space/venv/lib/python3.9/site-packages/ray/tune/registry.py in register(self, category, key, value)
    148             raise TuneError("Unknown category {} not among {}".format(
    149                 category, KNOWN_CATEGORIES))
--> 150         self._to_flush[(category, key)] = pickle.dumps_debug(value)
    151         if _internal_kv_initialized():
    152             self.flush_values()

/napi/k_working_dir/ds_test/shared-space/venv/lib/python3.9/site-packages/ray/cloudpickle/__init__.py in dumps_debug(obj, *args, **kwargs)
     37 def dumps_debug(obj, *args, **kwargs):
     38     try:
---> 39         return dumps(obj, *args, **kwargs)
     40     except (TypeError, PicklingError) as exc:
     41         if os.environ.get("RAY_PICKLE_VERBOSE_DEBUG"):

/napi/k_working_dir/ds_test/shared-space/venv/lib/python3.9/site-packages/ray/cloudpickle/cloudpickle_fast.py in dumps(obj, protocol, buffer_callback)
     71                 file, protocol=protocol, buffer_callback=buffer_callback
     72             )
---> 73             cp.dump(obj)
     74             return file.getvalue()
     75 

/napi/k_working_dir/ds_test/shared-space/venv/lib/python3.9/site-packages/ray/cloudpickle/cloudpickle_fast.py in dump(self, obj)
    618     def dump(self, obj):
    619         try:
--> 620             return Pickler.dump(self, obj)
    621         except RuntimeError as e:
    622             if "recursion" in e.args[0]:

/napi/k_working_dir/ds_test/shared-space/venv/lib/python3.9/site-packages/tensorflow/python/framework/ops.py in __reduce__(self)
   1072 
   1073   def __reduce__(self):
-> 1074     return convert_to_tensor, (self._numpy(),)
   1075 
   1076   def __copy__(self):

/napi/k_working_dir/ds_test/shared-space/venv/lib/python3.9/site-packages/tensorflow/python/framework/ops.py in _numpy(self)
   1115       return self._numpy_internal()
   1116     except core._NotOkStatusException as e:  # pylint: disable=protected-access
-> 1117       raise core._status_to_exception(e) from None  # pylint: disable=protected-access
   1118 
   1119   @property

InvalidArgumentError: Cannot convert a Tensor of dtype variant to a NumPy array.

Your help will be much appreciated!

[sonichi]

This appears to be a problem in ray's pickling. Could you try removing use_ray=True?
@Yard1 Have you got any idea?

[DavidNP96]

@wjlgatech. I'm running into this issue atm. Do you remember how you fixed this back then?