Autogluon: [Bug][HPO] CUDA does not support fork so you cannot AutoGluon HPO if you've initialized CUDA

Created on 1 Aug 2020 · 6Comments · Source: awslabs/autogluon

CUDA does not support fork: See PyTorch documentation (https://pytorch.org/docs/stable/notes/multiprocessing.html#cuda-in-multiprocessing). Thus, if we initialize the cuda before calling the autogluon scheduler, we will see an error: CUDA: initialization error

Example 1 (runnable)

pip install -U --pre "mxnet_cu100>=1.7.0b20200713, <2.0.0" -f https://sxjscience.github.io/KDD2020/

import numpy as np
import autogluon as ag
import mxnet as mx
from mxnet.gluon import nn, Trainer
from mxnet.util import use_np


@use_np
class Net:
    def train_fn(self, args, reporter):
        gpu_ctx_l = [mx.gpu(i) for i in range(mx.context.num_gpus())]
        print('num_gpus:', len(gpu_ctx_l))
        np.random.seed(123)
        mx.random.seed(123)
        net = nn.HybridSequential()
        net.add(nn.Dense(16))
        net.add(nn.Activation('relu'))
        net.add(nn.Dense(4))
        net.hybridize()
        net.initialize(ctx=gpu_ctx_l)
        trainer = Trainer(net.collect_params(), 'adam')
        for i in range(100):
            with mx.autograd.record():
                data = mx.np.random.normal(0, 1, (8, 4), ctx=gpu_ctx_l[0])
                out = net(data)
                loss = mx.np.square(out - data).sum()
                loss.backward()
                reporter(loss=loss.asnumpy().item(), iteration=i)
            trainer.step(1.0)



def run_tuning_jobs(fn, search_space):
    args_decorator = ag.args(**search_space)
    scheduler = ag.scheduler.FIFOScheduler(args_decorator(fn),
                                       resource={'num_cpus': 4, 'num_gpus': 1},
                                       num_trials=20,
                                       reward_attr='loss',
                                       time_attr='iteration')
    scheduler.run()
    scheduler.join_jobs()
    return scheduler


search_space = {
    'num_hidden': ag.space.Int(16, 32),
    'lr': ag.space.Real(1e-3, 1e-2)
}

net = Net()

scheduler = run_tuning_jobs(net.train_fn, search_space)

Example 2 (Raise error)

import numpy as np
import autogluon as ag
import mxnet as mx
from mxnet.gluon import nn, Trainer
from mxnet.util import use_np


@use_np
class Net:
    def train_fn(self, args, reporter):
        gpu_ctx_l = [mx.gpu(i) for i in range(mx.context.num_gpus())]
        print('num_gpus:', len(gpu_ctx_l))
        np.random.seed(123)
        mx.random.seed(123)
        net = nn.HybridSequential()
        net.add(nn.Dense(16))
        net.add(nn.Activation('relu'))
        net.add(nn.Dense(4))
        net.hybridize()
        net.initialize(ctx=gpu_ctx_l)
        trainer = Trainer(net.collect_params(), 'adam')
        for i in range(100):
            with mx.autograd.record():
                data = mx.np.random.normal(0, 1, (8, 4), ctx=gpu_ctx_l[0])
                out = net(data)
                loss = mx.np.square(out - data).sum()
                loss.backward()
                reporter(loss=loss.asnumpy().item(), iteration=i)
            trainer.step(1.0)



def run_tuning_jobs(fn, search_space):
    args_decorator = ag.args(**search_space)
    scheduler = ag.scheduler.FIFOScheduler(args_decorator(fn),
                                       resource={'num_cpus': 4, 'num_gpus': 1},
                                       num_trials=20,
                                       reward_attr='loss',
                                       time_attr='iteration')
    scheduler.run()
    scheduler.join_jobs()
    return scheduler


search_space = {
    'num_hidden': ag.space.Int(16, 32),
    'lr': ag.space.Real(1e-3, 1e-2)
}

net = Net()

# Add one line
a = mx.np.ones((10,), ctx=mx.gpu())
scheduler = run_tuning_jobs(net.train_fn, search_space)

bug

Source

sxjscience

All 6 comments

For Ray/Tune, we are able to do this:
@zhreshold @szha @jwmueller @Innixma @Jerryzcn

import numpy as np
import autogluon as ag
import ray
from ray import tune
import mxnet as mx
from mxnet.gluon import nn, Trainer
from mxnet.util import use_np

def get_mxnet_visible_gpus():
    """Get the number of GPUs that are visible to MXNet.

    Returns
    -------
    ctx_l
        The ctx list
    """
    import mxnet as mx
    gpu_count = 0
    while True:
        try:
            arr = mx.np.array(1.0, ctx=mx.gpu(gpu_count))
            arr.asnumpy()
            gpu_count += 1
        except Exception:
            break
    return [mx.gpu(i) for i in range(gpu_count)]


@use_np
class Net:
    def train_fn(self, args, reporter):
        np.random.seed(123)
        mx.random.seed(123)
        gpu_ctx_l = get_mxnet_visible_gpus()
        print(gpu_ctx_l)
        net = nn.HybridSequential()
        net.add(nn.Dense(args['num_hidden']))
        net.add(nn.Activation('relu'))
        net.add(nn.Dense(4))
        net.hybridize()
        net.initialize(ctx=gpu_ctx_l)
        trainer = Trainer(net.collect_params(), 'adam', {'learning_rate': args['lr']})
        for i in range(10):
            with mx.autograd.record():
                loss_l = []
                for ctx in gpu_ctx_l:
                    data = mx.np.random.normal(0, 1, (8, 4), ctx=ctx)
                    out = net(data)
                    loss = mx.np.square(out - data).sum()
                    loss_l.append(loss)
                for loss in loss_l:
                    loss.backward()
            sum_loss = sum([loss.asnumpy() for loss in loss_l])
            reporter(loss=-sum_loss, iteration=i)
            trainer.step(1.0)
        return net


search_space = {
    'num_hidden': tune.sample_from(lambda _: np.random.randint(16, 32)),
    'lr': tune.sample_from(lambda _: np.random.uniform(1e-3, 1e-2))
}

a = mx.np.ones((10,), ctx=mx.gpu())
net = Net()
analysis = tune.run(net.train_fn, config=search_space, num_samples=16, resources_per_trial={'gpu': 2})
print(analysis.dataframe())

sxjscience on 1 Aug 2020

👀1

Also @sxjscience pointed out the reason tasks like ImageClassification get around this issue is their task.fit() returns models to the CPU instead of GPU, which seems undesirable...

https://github.com/awslabs/autogluon/blob/5e1acab422289921ae9f7112e71855c2ea89e3b1/autogluon/task/image_classification/image_classification.py#L316

jwmueller on 2 Aug 2020

👍1

@sxjscience any clue how ray tune bypassed the issue? It's unclear to me how global imported module is still available while cuda array is not.

zhreshold on 2 Aug 2020

@zhreshold It seems that ray tune does not have the issue even if we are reusing the global variable. The following runs well in ray/tune.

import numpy as np
import autogluon as ag
import ray
from ray import tune
import torch as th
import torch.nn as nn


def objective(step, alpha, beta):
    return (0.1 + alpha)

class Net(nn.Module):
    def __init__(self, in_units, num_hidden):
        super().__init__()
        self.linear1 = nn.Linear(in_units, num_hidden)
        self.out_layer = nn.Linear(num_hidden, 1)

    def forward(self, x):
        return self.out_layer(nn.ReLU()(self.linear1(x)))


def train_nn(config):
    global a_gpu
    num_hidden = config['num_hidden']
    lr = config['lr']
    batch_size, in_units = 32, 8
    net = Net(in_units=in_units, num_hidden=num_hidden)
    net.cuda()
    net.train()
    optimizer = th.optim.Adam(net.parameters(), lr=lr, amsgrad=True)
    for i in range(10):
        optimizer.zero_grad()
        random_data_in = th.normal(0, 1, (batch_size, in_units)).cuda()
        random_data_out = 5 * random_data_in ** 2 + 1
        pred_out = net(random_data_in)
        loss = ((random_data_out - pred_out)**2).mean()
        loss.backward()
        with th.no_grad():
            print(a_gpu)
            print(loss.cpu().detach().numpy())
            tune.report(mean_loss=loss.cpu().detach().numpy())
        optimizer.step()


search_space = {
    'num_hidden': tune.sample_from(lambda _: np.random.randint(16, 32)),
    'lr': tune.sample_from(lambda _: np.random.uniform(1e-3, 1e-2))
}

# Create a tensor in GPU
global a_gpu
a_gpu = th.ones((10,)).cuda()
analysis = tune.run(train_nn, config=search_space, num_samples=4, resources_per_trial={'gpu': 1})
print(analysis.dataframe(metric='mean_loss', mode='min'))
print("Best config: ", analysis.get_best_config(metric="mean_loss", mode="min"))

b_gpu = th.ones((10,)).cuda()

print(b_gpu + a_gpu)

sxjscience on 8 Nov 2020

@gradientsky Pinging in case this is of interest to you

Innixma on 9 Nov 2020

Update on the reproducible example with latest AutoGluon after refactory:

import numpy as np
import autogluon as ag
import autogluon.core
import mxnet as mx
from mxnet.gluon import nn, Trainer
from mxnet.util import use_np


@use_np
class Net:
    def train_fn(self, args, reporter):
        gpu_ctx_l = [mx.gpu(i) for i in range(mx.context.num_gpus())]
        print('num_gpus:', len(gpu_ctx_l))
        np.random.seed(123)
        mx.random.seed(123)
        net = nn.HybridSequential()
        net.add(nn.Dense(16))
        net.add(nn.Activation('relu'))
        net.add(nn.Dense(4))
        net.hybridize()
        net.initialize(ctx=gpu_ctx_l)
        trainer = Trainer(net.collect_params(), 'adam')
        for i in range(100):
            with mx.autograd.record():
                data = mx.np.random.normal(0, 1, (8, 4), ctx=gpu_ctx_l[0])
                out = net(data)
                loss = mx.np.square(out - data).sum()
                loss.backward()
                reporter(loss=loss.asnumpy().item(), iteration=i)
            trainer.step(1.0)



def run_tuning_jobs(fn, search_space):
    args_decorator = ag.core.args(**search_space)
    scheduler = ag.core.scheduler.FIFOScheduler(args_decorator(fn),
                                       resource={'num_cpus': 4, 'num_gpus': 1},
                                       num_trials=20,
                                       reward_attr='loss',
                                       time_attr='iteration')
    scheduler.run()
    scheduler.join_jobs()
    return scheduler


search_space = {
    'num_hidden': ag.core.space.Int(16, 32),
    'lr': ag.core.space.Real(1e-3, 1e-2)
}

net = Net()

# Add one line
a = mx.np.ones((10,), ctx=mx.gpu())
scheduler = run_tuning_jobs(net.train_fn, search_space)

My guess is that switching to forkserver can help solve the problem.

sxjscience on 9 Nov 2020

Was this page helpful?

0 / 5 - 0 ratings