1

我有一个芹菜任务，需要 1 分钟到 2 天才能完成。我将 celery 作为守护进程模式运行，如下所示：

**/etc/default**
# celeryd

# A single node named w1
# If you want more worker nodes on the same machine list their names here:
#    CELERYD_NODES="w1 w2 w3"
CELERYD_NODES="w1"

# 5 minute hard time limit for tasks + enable events at startup.
CELERYD_OPTS="--time-limit=7200 -E"

# Full path to the Django project directory.
CELERYD_CHDIR="/home/zurelsoft/Documents/workspace/genalytics"

CELERYD_MULTI="$CELERYD_CHDIR/manage.py celeryd_multi"

# Logfiles are in /var/log/celery and must be
# writable by $CELERYD_USER
#     $ mkdir -p /var/log/celery
#     $ chown $CELERYD_USER:$CELERYD_GROUP /var/log/celery
#
CELERYD_LOG_FILE=/var/log/celery/%n.log

# Pidfiles are in /var/run/celery and must be
# writeable by $CELERYD_USER
#
#   $ mkdir -p /var/run/celery
#   $ chown $CELERYD_USER:$CELERYD_GROUP /var/run/celery
CELERYD_PID_FILE=/var/run/celery/%n.pid

# To create this user/group do:
#      $ addgroup --system celery
#      $ adduser --system --ingroup celery --no-create-home --disabled-password celery
CELERYD_USER="username"
CELERYD_GROUP="password"

/etc/init.d/celeryd的内容 

#!/bin/sh -e
# ============================================
#  celeryd - Starts the Celery worker daemon.
# ============================================
#
# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status}
# :Configuration file: /etc/default/celeryd
#
# See http://docs.celeryq.org/en/latest/cookbook/daemonizing.html#init-script-celeryd


### BEGIN INIT INFO
# Provides:          celeryd
# Required-Start:    $network $local_fs $remote_fs
# Required-Stop:     $network $local_fs $remote_fs
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: celery task worker daemon
### END INIT INFO

#set -e

DEFAULT_PID_FILE="/var/run/celeryd@%n.pid"
DEFAULT_LOG_FILE="/var/log/celeryd@%n.log"
DEFAULT_LOG_LEVEL="INFO"
DEFAULT_NODES="celery"
DEFAULT_CELERYD="-m celery.bin.celeryd_detach"

# /etc/init.d/celeryd: start and stop the celery task worker daemon.

CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/default/celeryd"}

test -f "$CELERY_DEFAULTS" && . "$CELERY_DEFAULTS"
if [ -f "/etc/default/celeryd" ]; then
    . /etc/default/celeryd
fi

CELERYD_PID_FILE=${CELERYD_PID_FILE:-${CELERYD_PIDFILE:-$DEFAULT_PID_FILE}}
CELERYD_LOG_FILE=${CELERYD_LOG_FILE:-${CELERYD_LOGFILE:-$DEFAULT_LOG_FILE}}
CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}}
CELERYD_MULTI=${CELERYD_MULTI:-"celeryd-multi"}
CELERYD=${CELERYD:-$DEFAULT_CELERYD}
CELERYCTL=${CELERYCTL:="celeryctl"}
CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES}

export CELERY_LOADER

if [ -n "$2" ]; then
    CELERYD_OPTS="$CELERYD_OPTS $2"
fi

CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE`
CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE`
if [ ! -d "$CELERYD_LOG_DIR" ]; then
    mkdir -p $CELERYD_LOG_DIR
fi
if [ ! -d "$CELERYD_PID_DIR" ]; then
    mkdir -p $CELERYD_PID_DIR
fi

# Extra start-stop-daemon options, like user/group.
if [ -n "$CELERYD_USER" ]; then
    DAEMON_OPTS="$DAEMON_OPTS --uid=$CELERYD_USER"
    chown "$CELERYD_USER" $CELERYD_LOG_DIR $CELERYD_PID_DIR
fi
if [ -n "$CELERYD_GROUP" ]; then
    DAEMON_OPTS="$DAEMON_OPTS --gid=$CELERYD_GROUP"
    chgrp "$CELERYD_GROUP" $CELERYD_LOG_DIR $CELERYD_PID_DIR
fi

if [ -n "$CELERYD_CHDIR" ]; then
    DAEMON_OPTS="$DAEMON_OPTS --workdir=\"$CELERYD_CHDIR\""
fi


check_dev_null() {
    if [ ! -c /dev/null ]; then
        echo "/dev/null is not a character device!"
        exit 1
    fi
}

ensure_dir() {
    if [ -d "$1" ]; then
        mkdir -p "$1"
        chown $CELERYD_USER:$CELERYD_GROUP "$1"
        chmod 02755 "$1"
    fi
}

check_paths() {
    ensure_dir "$(dirname $CELERYD_PID_FILE)"
    ensure_dir "$(dirname $CELERYD_LOG_FILE)"
}

export PATH="${PATH:+$PATH:}/usr/sbin:/sbin"


stop_workers () {
    $CELERYD_MULTI stop $CELERYD_NODES --pidfile="$CELERYD_PID_FILE"
}


start_workers () {
    $CELERYD_MULTI start $CELERYD_NODES $DAEMON_OPTS        \
                         --pidfile="$CELERYD_PID_FILE"      \
                         --logfile="$CELERYD_LOG_FILE"      \
                         --loglevel="$CELERYD_LOG_LEVEL"    \
                         --cmd="$CELERYD"                   \
                         $CELERYD_OPTS
}


restart_workers () {
    $CELERYD_MULTI restart $CELERYD_NODES $DAEMON_OPTS      \
                           --pidfile="$CELERYD_PID_FILE"    \
                           --logfile="$CELERYD_LOG_FILE"    \
                           --loglevel="$CELERYD_LOG_LEVEL"  \
                           --cmd="$CELERYD"                 \
                           $CELERYD_OPTS
}



case "$1" in
    start)
        check_dev_null
        check_paths
        start_workers
    ;;

    stop)
        check_dev_null
        check_paths
        stop_workers
    ;;

    reload|force-reload)
        echo "Use restart"
    ;;

    status)
        $CELERYCTL status $CELERYCTL_OPTS
    ;;

    restart)
        check_dev_null
        check_paths
        restart_workers
    ;;

    try-restart)
        check_dev_null
        check_paths
        restart_workers
    ;;

    *)
        echo "Usage: /etc/init.d/celeryd {start|stop|restart|try-restart|kill}"
        exit 1
    ;;
esac

exit 0

可能同时有多个任务。我在日志文件中收到此错误：

TimeLimitExceeded(300.0,)
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/billiard/pool.py", line 516, in on_hard_timeout
    raise TimeLimitExceeded(job._timeout)
TimeLimitExceeded: TimeLimitExceeded(300.0,)
[2013-06-06 07:37:41,795: ERROR/MainProcess] Hard time limit (300.0s) exceeded for fileupload.tasks.file_transfer[a2851668-e563-4276-a256-0705fc6de1ef]
[2013-06-06 07:37:42,028: ERROR/MainProcess] Process 'PoolWorker-2' pid:25050 exited with exitcode 15

当我尝试启动任务时，我在 Django 中收到以下错误：

2013-06-06 09:09:24,940: ERROR/MainProcess] Unrecoverable error: OperationalError("(OperationalError) (1053, 'Server shutdown in progress')",)
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/celery/worker/__init__.py", line 363, in start
    component.start()

如果我重新启动两个服务器，它可以正常工作 3-4 小时，然后我会收到同样的错误。怎么了？

4

0 回答 0