perfectqueue 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ChangeLog +7 -0
- data/README.rdoc +23 -23
- data/lib/perfectqueue/backend/simpledb.rb +10 -10
- data/lib/perfectqueue/command/perfectqueue.rb +44 -30
- data/lib/perfectqueue/version.rb +1 -1
- data/lib/perfectqueue/worker.rb +31 -16
- metadata +3 -3
data/ChangeLog
CHANGED
data/README.rdoc
CHANGED
@@ -37,14 +37,14 @@ PerfectQueue uses following database schema:
|
|
37
37
|
$ perfectqueue \
|
38
38
|
--database mysql://user:password@localhost/mydb \
|
39
39
|
--table perfectqueue \
|
40
|
-
--push unique-key-id
|
40
|
+
--push unique-key-id '{"any":"data"}'
|
41
41
|
|
42
42
|
# SimpleDB
|
43
43
|
$ perfectqueue \
|
44
44
|
--simpledb your-simpledb-domain-name \
|
45
45
|
-k AWS_KEY_ID \
|
46
46
|
-s AWS_SECRET_KEY \
|
47
|
-
--push unique-key-id
|
47
|
+
--push unique-key-id '{"any":"data"}'
|
48
48
|
|
49
49
|
*Using* *PerfectQueue* *library:*
|
50
50
|
|
@@ -118,33 +118,33 @@ Use _perfectqueue_ command to execute a command.
|
|
118
118
|
|
119
119
|
Usage: perfectqueue [options] [-- <ARGV-for-exec-or-run>]
|
120
120
|
|
121
|
-
--push ID
|
121
|
+
--push <ID> <DATA> Push a task to the queue
|
122
122
|
--list Show queued tasks
|
123
|
-
--cancel ID
|
124
|
-
--configure PATH.yaml
|
123
|
+
--cancel <ID> Cancel a queued task
|
124
|
+
--configure <PATH.yaml> Write configuration file
|
125
125
|
|
126
|
-
--exec COMMAND
|
127
|
-
--run SCRIPT.rb
|
126
|
+
--exec <COMMAND> Execute command
|
127
|
+
--run <SCRIPT.rb> Run method named 'run' defined in the script
|
128
128
|
|
129
|
-
-f, --file PATH.yaml
|
129
|
+
-f, --file <PATH.yaml> Read configuration file
|
130
130
|
-C, --run-class Class name for --run (default: ::Run)
|
131
|
-
-t, --timeout SEC
|
132
|
-
-b, --heartbeat-interval SEC
|
133
|
-
-x, --kill-timeout SEC
|
134
|
-
-X, --kill-interval SEC
|
135
|
-
-i, --poll-interval SEC
|
136
|
-
-r, --retry-wait SEC
|
137
|
-
-e, --expire SEC
|
131
|
+
-t, --timeout <SEC> Time for another worker to take over a task when this worker goes down (default: 600)
|
132
|
+
-b, --heartbeat-interval <SEC> Threshold time to extend the timeout (heartbeat interval) (default: timeout * 3/4)
|
133
|
+
-x, --kill-timeout <SEC> Threshold time to kill a task process (default: timeout * 10)
|
134
|
+
-X, --kill-interval <SEC> Threshold time to retry killing a task process (default: 60)
|
135
|
+
-i, --poll-interval <SEC> Polling interval (default: 1)
|
136
|
+
-r, --retry-wait <SEC> Time to retry a task when it is failed (default: same as timeout)
|
137
|
+
-e, --expire <SEC> Threshold time to expire a task (default: 345601 (4days))
|
138
138
|
|
139
|
-
--database URI
|
140
|
-
--table NAME
|
141
|
-
--simpledb DOMAIN
|
142
|
-
-k, --key-id ID
|
143
|
-
-s, --secret-key KEY
|
139
|
+
--database <URI> Use RDBMS for the backend database (e.g.: mysql://user:password@localhost/mydb)
|
140
|
+
--table <NAME> backend: name of the table (default: perfectqueue)
|
141
|
+
--simpledb <DOMAIN> Use Amazon SimpleDB for the backend database (e.g.: --simpledb mydomain -k KEY_ID -s SEC_KEY)
|
142
|
+
-k, --key-id <ID> AWS Access Key ID
|
143
|
+
-s, --secret-key <KEY> AWS Secret Access Key
|
144
144
|
|
145
|
-
-w, --worker NUM
|
146
|
-
-d, --daemon PIDFILE
|
147
|
-
-o, --log PATH
|
145
|
+
-w, --worker <NUM> Number of worker threads (default: 1)
|
146
|
+
-d, --daemon <PIDFILE> Daemonize (default: foreground)
|
147
|
+
-o, --log <PATH> log file path
|
148
148
|
-v, --verbose verbose mode
|
149
149
|
|
150
150
|
|
@@ -27,16 +27,16 @@ class SimpleDBBackend < Backend
|
|
27
27
|
end
|
28
28
|
|
29
29
|
def list(&block)
|
30
|
-
@domain.items.
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
30
|
+
@domain.items.select('timeout', 'data', 'created_at',
|
31
|
+
:where => "created_at != '' AND timeout > '#{int_encode(0)}'",
|
32
|
+
:order => [:timeout, :asc],
|
33
|
+
:consistent_read => @consistent_read) {|itemdata|
|
34
|
+
id = itemdata.name
|
35
|
+
attrs = itemdata.attributes
|
36
|
+
created_at = int_decode(attrs['created_at'].first)
|
37
|
+
data = attrs['data'].first
|
38
|
+
timeout = int_decode(attrs['timeout'].first)
|
39
|
+
yield id, created_at, data, timeout
|
40
40
|
}
|
41
41
|
end
|
42
42
|
|
@@ -31,108 +31,108 @@ op.on('-v', '--verbose', "verbose mode", TrueClass) {|b|
|
|
31
31
|
}
|
32
32
|
op.separator("")
|
33
33
|
|
34
|
-
op.on('--push ID
|
34
|
+
op.on('--push <ID> <DATA>', 'Push a task to the queue') {|s|
|
35
35
|
type = :push
|
36
|
-
id
|
36
|
+
id = s
|
37
37
|
}
|
38
38
|
|
39
39
|
op.on('--list', 'Show queued tasks', TrueClass) {|b|
|
40
40
|
type = :list
|
41
41
|
}
|
42
42
|
|
43
|
-
op.on('--cancel ID', 'Cancel a queued task') {|s|
|
43
|
+
op.on('--cancel <ID>', 'Cancel a queued task') {|s|
|
44
44
|
type = :cancel
|
45
45
|
id = s
|
46
46
|
}
|
47
47
|
|
48
|
-
op.on('--configure PATH.yaml', 'Write configuration file') {|s|
|
48
|
+
op.on('--configure <PATH.yaml>', 'Write configuration file') {|s|
|
49
49
|
type = :conf
|
50
50
|
confout = s
|
51
51
|
}
|
52
52
|
|
53
53
|
op.separator("")
|
54
54
|
|
55
|
-
op.on('--exec COMMAND', 'Execute command') {|s|
|
55
|
+
op.on('--exec <COMMAND>', 'Execute command') {|s|
|
56
56
|
type = :exec
|
57
57
|
conf[:exec] = s
|
58
58
|
}
|
59
59
|
|
60
|
-
op.on('--run SCRIPT.rb', 'Run method named \'run\' defined in the script') {|s|
|
60
|
+
op.on('--run <SCRIPT.rb>', 'Run method named \'run\' defined in the script') {|s|
|
61
61
|
type = :run
|
62
62
|
conf[:run] = s
|
63
63
|
}
|
64
64
|
|
65
65
|
op.separator("")
|
66
66
|
|
67
|
-
op.on('-f', '--file PATH.yaml', 'Read configuration file') {|s|
|
68
|
-
conf[:
|
67
|
+
op.on('-f', '--file <PATH.yaml>', 'Read configuration file') {|s|
|
68
|
+
(conf[:files] ||= []) << s
|
69
69
|
}
|
70
70
|
|
71
71
|
op.on('-C', '--run-class', 'Class name for --run (default: ::Run)') {|s|
|
72
72
|
conf[:run_class] = s
|
73
73
|
}
|
74
74
|
|
75
|
-
op.on('-t', '--timeout SEC', 'Time for another worker to take over a task when this worker goes down (default: 600)', Integer) {|i|
|
75
|
+
op.on('-t', '--timeout <SEC>', 'Time for another worker to take over a task when this worker goes down (default: 600)', Integer) {|i|
|
76
76
|
conf[:timeout] = i
|
77
77
|
}
|
78
78
|
|
79
|
-
op.on('-b', '--heartbeat-interval SEC', 'Threshold time to extend the timeout (heartbeat interval) (default: timeout * 3/4)', Integer) {|i|
|
79
|
+
op.on('-b', '--heartbeat-interval <SEC>', 'Threshold time to extend the timeout (heartbeat interval) (default: timeout * 3/4)', Integer) {|i|
|
80
80
|
conf[:heartbeat_interval] = i
|
81
81
|
}
|
82
82
|
|
83
|
-
op.on('-x', '--kill-timeout SEC', 'Threshold time to kill a task process (default: timeout * 10)', Integer) {|i|
|
83
|
+
op.on('-x', '--kill-timeout <SEC>', 'Threshold time to kill a task process (default: timeout * 10)', Integer) {|i|
|
84
84
|
conf[:kill_timeout] = i
|
85
85
|
}
|
86
86
|
|
87
|
-
op.on('-X', '--kill-interval SEC', 'Threshold time to retry killing a task process (default: 60)', Integer) {|i|
|
87
|
+
op.on('-X', '--kill-interval <SEC>', 'Threshold time to retry killing a task process (default: 60)', Integer) {|i|
|
88
88
|
conf[:kill_interval] = i
|
89
89
|
}
|
90
90
|
|
91
|
-
op.on('-i', '--poll-interval SEC', 'Polling interval (default: 1)', Integer) {|i|
|
91
|
+
op.on('-i', '--poll-interval <SEC>', 'Polling interval (default: 1)', Integer) {|i|
|
92
92
|
conf[:poll_interval] = i
|
93
93
|
}
|
94
94
|
|
95
|
-
op.on('-r', '--retry-wait SEC', 'Time to retry a task when it is failed (default: same as timeout)', Integer) {|i|
|
95
|
+
op.on('-r', '--retry-wait <SEC>', 'Time to retry a task when it is failed (default: same as timeout)', Integer) {|i|
|
96
96
|
conf[:retry_wait] = i
|
97
97
|
}
|
98
98
|
|
99
|
-
op.on('-e', '--expire SEC', 'Threshold time to expire a task (default:
|
99
|
+
op.on('-e', '--expire <SEC>', 'Threshold time to expire a task (default: 345601 (4days))', Integer) {|i|
|
100
100
|
conf[:expire] = i
|
101
101
|
}
|
102
102
|
|
103
103
|
op.separator("")
|
104
104
|
|
105
|
-
op.on('--database URI', 'Use RDBMS for the backend database (e.g.: mysql://user:password@localhost/mydb)') {|s|
|
105
|
+
op.on('--database <URI>', 'Use RDBMS for the backend database (e.g.: mysql://user:password@localhost/mydb)') {|s|
|
106
106
|
conf[:backend_database] = s
|
107
107
|
}
|
108
108
|
|
109
|
-
op.on('--table NAME', 'backend: name of the table (default: perfectqueue)') {|s|
|
109
|
+
op.on('--table <NAME>', 'backend: name of the table (default: perfectqueue)') {|s|
|
110
110
|
conf[:backend_table] = s
|
111
111
|
}
|
112
112
|
|
113
|
-
op.on('--simpledb DOMAIN', 'Use Amazon SimpleDB for the backend database (e.g.: --simpledb mydomain -k KEY_ID -s SEC_KEY)') {|s|
|
113
|
+
op.on('--simpledb <DOMAIN>', 'Use Amazon SimpleDB for the backend database (e.g.: --simpledb mydomain -k KEY_ID -s SEC_KEY)') {|s|
|
114
114
|
conf[:backend_simpledb] = s
|
115
115
|
}
|
116
116
|
|
117
|
-
op.on('-k', '--key-id ID', 'AWS Access Key ID') {|s|
|
117
|
+
op.on('-k', '--key-id <ID>', 'AWS Access Key ID') {|s|
|
118
118
|
conf[:backend_key_id] = s
|
119
119
|
}
|
120
120
|
|
121
|
-
op.on('-s', '--secret-key KEY', 'AWS Secret Access Key') {|s|
|
121
|
+
op.on('-s', '--secret-key <KEY>', 'AWS Secret Access Key') {|s|
|
122
122
|
conf[:backend_secret_key] = s
|
123
123
|
}
|
124
124
|
|
125
125
|
op.separator("")
|
126
126
|
|
127
|
-
op.on('-w', '--worker NUM', 'Number of worker threads (default: 1)', Integer) {|i|
|
127
|
+
op.on('-w', '--worker <NUM>', 'Number of worker threads (default: 1)', Integer) {|i|
|
128
128
|
conf[:workers] = i
|
129
129
|
}
|
130
130
|
|
131
|
-
op.on('-d', '--daemon PIDFILE', 'Daemonize (default: foreground)') {|s|
|
131
|
+
op.on('-d', '--daemon <PIDFILE>', 'Daemonize (default: foreground)') {|s|
|
132
132
|
conf[:daemon] = s
|
133
133
|
}
|
134
134
|
|
135
|
-
op.on('-o', '--log PATH', "log file path") {|s|
|
135
|
+
op.on('-o', '--log <PATH>', "log file path") {|s|
|
136
136
|
conf[:log] = s
|
137
137
|
}
|
138
138
|
|
@@ -159,15 +159,29 @@ begin
|
|
159
159
|
end
|
160
160
|
op.parse!(argv)
|
161
161
|
|
162
|
-
|
163
|
-
|
162
|
+
case type
|
163
|
+
when :push
|
164
|
+
if argv.length != 1
|
165
|
+
usage nil
|
166
|
+
end
|
167
|
+
data = argv[0]
|
168
|
+
|
169
|
+
else
|
170
|
+
if argv.length != 0
|
171
|
+
usage nil
|
172
|
+
end
|
164
173
|
end
|
165
174
|
|
166
|
-
if conf[:
|
175
|
+
if conf[:files]
|
167
176
|
require 'yaml'
|
168
|
-
|
177
|
+
docs = ''
|
178
|
+
conf[:files].each {|file|
|
179
|
+
docs << File.read(file)
|
180
|
+
}
|
169
181
|
y = {}
|
170
|
-
|
182
|
+
YAML.load_documents(docs) {|yaml|
|
183
|
+
yaml.each_pair {|k,v| y[k.to_sym] = v }
|
184
|
+
}
|
171
185
|
|
172
186
|
conf = defaults.merge(y).merge(conf)
|
173
187
|
|
@@ -265,7 +279,7 @@ when :cancel
|
|
265
279
|
if canceled
|
266
280
|
puts "Task id=#{id} is canceled."
|
267
281
|
else
|
268
|
-
puts "Task id=#{id} does not exist."
|
282
|
+
puts "Task id=#{id} does not exist. abort"
|
269
283
|
end
|
270
284
|
|
271
285
|
when :push
|
@@ -273,7 +287,7 @@ when :push
|
|
273
287
|
if submitted
|
274
288
|
puts "Task id=#{id} is submitted."
|
275
289
|
else
|
276
|
-
puts "Task id=#{id}
|
290
|
+
puts "Task id=#{id} is duplicated. abort."
|
277
291
|
end
|
278
292
|
|
279
293
|
when :exec, :run
|
data/lib/perfectqueue/version.rb
CHANGED
data/lib/perfectqueue/worker.rb
CHANGED
@@ -37,28 +37,32 @@ class MonitorThread
|
|
37
37
|
@cond.wait(@mutex)
|
38
38
|
end
|
39
39
|
}
|
40
|
-
|
41
|
-
sleep 1
|
42
|
-
@mutex.synchronize {
|
43
|
-
return if @engine.finished?
|
44
|
-
break unless @token
|
45
|
-
now = Time.now.to_i
|
46
|
-
try_extend(now)
|
47
|
-
try_kill(now)
|
48
|
-
}
|
49
|
-
end
|
40
|
+
process
|
50
41
|
end
|
51
42
|
rescue
|
52
43
|
@engine.stop($!)
|
53
44
|
end
|
54
45
|
|
46
|
+
def process
|
47
|
+
while true
|
48
|
+
sleep 1
|
49
|
+
@mutex.synchronize {
|
50
|
+
return if @engine.finished?
|
51
|
+
return unless @token
|
52
|
+
now = Time.now.to_i
|
53
|
+
try_extend(now)
|
54
|
+
try_kill(now)
|
55
|
+
}
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
55
59
|
def try_extend(now)
|
56
60
|
if now >= @heartbeat_time && !@canceled
|
57
|
-
@log.debug "extending timeout=#{now+@timeout} id=#{@
|
61
|
+
@log.debug "extending timeout=#{now+@timeout} id=#{@task_id}"
|
58
62
|
begin
|
59
63
|
@backend.update(@token, now+@timeout)
|
60
64
|
rescue CanceledError
|
61
|
-
@log.info "task id=#{@
|
65
|
+
@log.info "task id=#{@task_id} is canceled."
|
62
66
|
@canceled = true
|
63
67
|
@kill_time = now
|
64
68
|
end
|
@@ -75,8 +79,15 @@ class MonitorThread
|
|
75
79
|
|
76
80
|
def kill!
|
77
81
|
if @kill_proc
|
78
|
-
@log.info "killing
|
79
|
-
|
82
|
+
@log.info "killing id=#{@task_id}..."
|
83
|
+
begin
|
84
|
+
@kill_proc.call
|
85
|
+
rescue
|
86
|
+
@log.info "kill failed id=#{@task_id}: #{$!}"
|
87
|
+
$!.backtrace.each {|bt|
|
88
|
+
$log.debug " #{bt}"
|
89
|
+
}
|
90
|
+
end
|
80
91
|
end
|
81
92
|
end
|
82
93
|
|
@@ -90,10 +101,11 @@ class MonitorThread
|
|
90
101
|
@thread.join
|
91
102
|
end
|
92
103
|
|
93
|
-
def set(token)
|
104
|
+
def set(token, task_id)
|
94
105
|
@mutex.synchronize {
|
95
106
|
now = Time.now.to_i
|
96
107
|
@token = token
|
108
|
+
@task_id = task_id
|
97
109
|
@heartbeat_time = now + @heartbeat_interval
|
98
110
|
@kill_time = now + @kill_timeout
|
99
111
|
@kill_proc = nil
|
@@ -165,7 +177,7 @@ class Worker
|
|
165
177
|
def process(token, task)
|
166
178
|
@log.info "processing task id=#{task.id}"
|
167
179
|
|
168
|
-
@monitor.set(token)
|
180
|
+
@monitor.set(token, task.id)
|
169
181
|
success = false
|
170
182
|
begin
|
171
183
|
run = @run_class.new(task)
|
@@ -181,6 +193,9 @@ class Worker
|
|
181
193
|
|
182
194
|
rescue
|
183
195
|
@log.info "failed id=#{task.id}: #{$!}"
|
196
|
+
$!.backtrace.each {|bt|
|
197
|
+
$log.debug " #{bt}"
|
198
|
+
}
|
184
199
|
|
185
200
|
ensure
|
186
201
|
@monitor.reset(success)
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perfectqueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 1
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 7
|
9
|
-
-
|
10
|
-
version: 0.7.
|
9
|
+
- 1
|
10
|
+
version: 0.7.1
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Sadayuki Furuhashi
|