我有一个芹菜任务,需要 1 分钟到 2 天才能完成。我将 celery 作为守护进程模式运行,如下所示:
**/etc/default**
# celeryd
# A single node named w1
# If you want more worker nodes on the same machine list their names here:
# CELERYD_NODES="w1 w2 w3"
CELERYD_NODES="w1"
# 5 minute hard time limit for tasks + enable events at startup.
CELERYD_OPTS="--time-limit=7200 -E"
# Full path to the Django project directory.
CELERYD_CHDIR="/home/zurelsoft/Documents/workspace/genalytics"
CELERYD_MULTI="$CELERYD_CHDIR/manage.py celeryd_multi"
# Logfiles are in /var/log/celery and must be
# writable by $CELERYD_USER
# $ mkdir -p /var/log/celery
# $ chown $CELERYD_USER:$CELERYD_GROUP /var/log/celery
#
CELERYD_LOG_FILE=/var/log/celery/%n.log
# Pidfiles are in /var/run/celery and must be
# writeable by $CELERYD_USER
#
# $ mkdir -p /var/run/celery
# $ chown $CELERYD_USER:$CELERYD_GROUP /var/run/celery
CELERYD_PID_FILE=/var/run/celery/%n.pid
# To create this user/group do:
# $ addgroup --system celery
# $ adduser --system --ingroup celery --no-create-home --disabled-password celery
CELERYD_USER="username"
CELERYD_GROUP="password"
/etc/init.d/celeryd的内容
#!/bin/sh -e
# ============================================
# celeryd - Starts the Celery worker daemon.
# ============================================
#
# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status}
# :Configuration file: /etc/default/celeryd
#
# See http://docs.celeryq.org/en/latest/cookbook/daemonizing.html#init-script-celeryd
### BEGIN INIT INFO
# Provides: celeryd
# Required-Start: $network $local_fs $remote_fs
# Required-Stop: $network $local_fs $remote_fs
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: celery task worker daemon
### END INIT INFO
#set -e
DEFAULT_PID_FILE="/var/run/celeryd@%n.pid"
DEFAULT_LOG_FILE="/var/log/celeryd@%n.log"
DEFAULT_LOG_LEVEL="INFO"
DEFAULT_NODES="celery"
DEFAULT_CELERYD="-m celery.bin.celeryd_detach"
# /etc/init.d/celeryd: start and stop the celery task worker daemon.
CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/default/celeryd"}
test -f "$CELERY_DEFAULTS" && . "$CELERY_DEFAULTS"
if [ -f "/etc/default/celeryd" ]; then
. /etc/default/celeryd
fi
CELERYD_PID_FILE=${CELERYD_PID_FILE:-${CELERYD_PIDFILE:-$DEFAULT_PID_FILE}}
CELERYD_LOG_FILE=${CELERYD_LOG_FILE:-${CELERYD_LOGFILE:-$DEFAULT_LOG_FILE}}
CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}}
CELERYD_MULTI=${CELERYD_MULTI:-"celeryd-multi"}
CELERYD=${CELERYD:-$DEFAULT_CELERYD}
CELERYCTL=${CELERYCTL:="celeryctl"}
CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES}
export CELERY_LOADER
if [ -n "$2" ]; then
CELERYD_OPTS="$CELERYD_OPTS $2"
fi
CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE`
CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE`
if [ ! -d "$CELERYD_LOG_DIR" ]; then
mkdir -p $CELERYD_LOG_DIR
fi
if [ ! -d "$CELERYD_PID_DIR" ]; then
mkdir -p $CELERYD_PID_DIR
fi
# Extra start-stop-daemon options, like user/group.
if [ -n "$CELERYD_USER" ]; then
DAEMON_OPTS="$DAEMON_OPTS --uid=$CELERYD_USER"
chown "$CELERYD_USER" $CELERYD_LOG_DIR $CELERYD_PID_DIR
fi
if [ -n "$CELERYD_GROUP" ]; then
DAEMON_OPTS="$DAEMON_OPTS --gid=$CELERYD_GROUP"
chgrp "$CELERYD_GROUP" $CELERYD_LOG_DIR $CELERYD_PID_DIR
fi
if [ -n "$CELERYD_CHDIR" ]; then
DAEMON_OPTS="$DAEMON_OPTS --workdir=\"$CELERYD_CHDIR\""
fi
check_dev_null() {
if [ ! -c /dev/null ]; then
echo "/dev/null is not a character device!"
exit 1
fi
}
ensure_dir() {
if [ -d "$1" ]; then
mkdir -p "$1"
chown $CELERYD_USER:$CELERYD_GROUP "$1"
chmod 02755 "$1"
fi
}
check_paths() {
ensure_dir "$(dirname $CELERYD_PID_FILE)"
ensure_dir "$(dirname $CELERYD_LOG_FILE)"
}
export PATH="${PATH:+$PATH:}/usr/sbin:/sbin"
stop_workers () {
$CELERYD_MULTI stop $CELERYD_NODES --pidfile="$CELERYD_PID_FILE"
}
start_workers () {
$CELERYD_MULTI start $CELERYD_NODES $DAEMON_OPTS \
--pidfile="$CELERYD_PID_FILE" \
--logfile="$CELERYD_LOG_FILE" \
--loglevel="$CELERYD_LOG_LEVEL" \
--cmd="$CELERYD" \
$CELERYD_OPTS
}
restart_workers () {
$CELERYD_MULTI restart $CELERYD_NODES $DAEMON_OPTS \
--pidfile="$CELERYD_PID_FILE" \
--logfile="$CELERYD_LOG_FILE" \
--loglevel="$CELERYD_LOG_LEVEL" \
--cmd="$CELERYD" \
$CELERYD_OPTS
}
case "$1" in
start)
check_dev_null
check_paths
start_workers
;;
stop)
check_dev_null
check_paths
stop_workers
;;
reload|force-reload)
echo "Use restart"
;;
status)
$CELERYCTL status $CELERYCTL_OPTS
;;
restart)
check_dev_null
check_paths
restart_workers
;;
try-restart)
check_dev_null
check_paths
restart_workers
;;
*)
echo "Usage: /etc/init.d/celeryd {start|stop|restart|try-restart|kill}"
exit 1
;;
esac
exit 0
可能同时有多个任务。我在日志文件中收到此错误:
TimeLimitExceeded(300.0,)
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/billiard/pool.py", line 516, in on_hard_timeout
raise TimeLimitExceeded(job._timeout)
TimeLimitExceeded: TimeLimitExceeded(300.0,)
[2013-06-06 07:37:41,795: ERROR/MainProcess] Hard time limit (300.0s) exceeded for fileupload.tasks.file_transfer[a2851668-e563-4276-a256-0705fc6de1ef]
[2013-06-06 07:37:42,028: ERROR/MainProcess] Process 'PoolWorker-2' pid:25050 exited with exitcode 15
当我尝试启动任务时,我在 Django 中收到以下错误:
2013-06-06 09:09:24,940: ERROR/MainProcess] Unrecoverable error: OperationalError("(OperationalError) (1053, 'Server shutdown in progress')",)
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/celery/worker/__init__.py", line 363, in start
component.start()
如果我重新启动两个服务器,它可以正常工作 3-4 小时,然后我会收到同样的错误。怎么了?