perfectqueue 0.7.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/ChangeLog CHANGED
@@ -1,4 +1,11 @@
1
1
 
2
+ == 2011-08-31 version 0.7.1
3
+
4
+ * Fixed kill behavior
5
+ * Supported to read multiple configuration files
6
+ * Optimized --list subcommand
7
+
8
+
2
9
  == 2011-08-31 version 0.7.0
3
10
 
4
11
  * Backend uses logical delete instead of deleting tasks when task is finished
data/README.rdoc CHANGED
@@ -37,14 +37,14 @@ PerfectQueue uses following database schema:
37
37
  $ perfectqueue \
38
38
  --database mysql://user:password@localhost/mydb \
39
39
  --table perfectqueue \
40
- --push unique-key-id='{"any":"data"}'
40
+ --push unique-key-id '{"any":"data"}'
41
41
 
42
42
  # SimpleDB
43
43
  $ perfectqueue \
44
44
  --simpledb your-simpledb-domain-name \
45
45
  -k AWS_KEY_ID \
46
46
  -s AWS_SECRET_KEY \
47
- --push unique-key-id='{"any":"data"}'
47
+ --push unique-key-id '{"any":"data"}'
48
48
 
49
49
  *Using* *PerfectQueue* *library:*
50
50
 
@@ -118,33 +118,33 @@ Use _perfectqueue_ command to execute a command.
118
118
 
119
119
  Usage: perfectqueue [options] [-- <ARGV-for-exec-or-run>]
120
120
 
121
- --push ID=DATA Push a task to the queue
121
+ --push <ID> <DATA> Push a task to the queue
122
122
  --list Show queued tasks
123
- --cancel ID Cancel a queued task
124
- --configure PATH.yaml Write configuration file
123
+ --cancel <ID> Cancel a queued task
124
+ --configure <PATH.yaml> Write configuration file
125
125
 
126
- --exec COMMAND Execute command
127
- --run SCRIPT.rb Run method named 'run' defined in the script
126
+ --exec <COMMAND> Execute command
127
+ --run <SCRIPT.rb> Run method named 'run' defined in the script
128
128
 
129
- -f, --file PATH.yaml Read configuration file
129
+ -f, --file <PATH.yaml> Read configuration file
130
130
  -C, --run-class Class name for --run (default: ::Run)
131
- -t, --timeout SEC Time for another worker to take over a task when this worker goes down (default: 600)
132
- -b, --heartbeat-interval SEC Threshold time to extend the timeout (heartbeat interval) (default: timeout * 3/4)
133
- -x, --kill-timeout SEC Threshold time to kill a task process (default: timeout * 10)
134
- -X, --kill-interval SEC Threshold time to retry killing a task process (default: 60)
135
- -i, --poll-interval SEC Polling interval (default: 1)
136
- -r, --retry-wait SEC Time to retry a task when it is failed (default: same as timeout)
137
- -e, --expire SEC Threshold time to expire a task (default: 345600 (4days))
131
+ -t, --timeout <SEC> Time for another worker to take over a task when this worker goes down (default: 600)
132
+ -b, --heartbeat-interval <SEC> Threshold time to extend the timeout (heartbeat interval) (default: timeout * 3/4)
133
+ -x, --kill-timeout <SEC> Threshold time to kill a task process (default: timeout * 10)
134
+ -X, --kill-interval <SEC> Threshold time to retry killing a task process (default: 60)
135
+ -i, --poll-interval <SEC> Polling interval (default: 1)
136
+ -r, --retry-wait <SEC> Time to retry a task when it is failed (default: same as timeout)
137
+ -e, --expire <SEC> Threshold time to expire a task (default: 345601 (4days))
138
138
 
139
- --database URI Use RDBMS for the backend database (e.g.: mysql://user:password@localhost/mydb)
140
- --table NAME backend: name of the table (default: perfectqueue)
141
- --simpledb DOMAIN Use Amazon SimpleDB for the backend database (e.g.: --simpledb mydomain -k KEY_ID -s SEC_KEY)
142
- -k, --key-id ID AWS Access Key ID
143
- -s, --secret-key KEY AWS Secret Access Key
139
+ --database <URI> Use RDBMS for the backend database (e.g.: mysql://user:password@localhost/mydb)
140
+ --table <NAME> backend: name of the table (default: perfectqueue)
141
+ --simpledb <DOMAIN> Use Amazon SimpleDB for the backend database (e.g.: --simpledb mydomain -k KEY_ID -s SEC_KEY)
142
+ -k, --key-id <ID> AWS Access Key ID
143
+ -s, --secret-key <KEY> AWS Secret Access Key
144
144
 
145
- -w, --worker NUM Number of worker threads (default: 1)
146
- -d, --daemon PIDFILE Daemonize (default: foreground)
147
- -o, --log PATH log file path
145
+ -w, --worker <NUM> Number of worker threads (default: 1)
146
+ -d, --daemon <PIDFILE> Daemonize (default: foreground)
147
+ -o, --log <PATH> log file path
148
148
  -v, --verbose verbose mode
149
149
 
150
150
 
@@ -27,16 +27,16 @@ class SimpleDBBackend < Backend
27
27
  end
28
28
 
29
29
  def list(&block)
30
- @domain.items.each {|item|
31
- id = item.name
32
- attrs = item.data.attributes
33
- salt = attrs['created_at'].first
34
- if salt && !salt.empty?
35
- created_at = int_decode(salt)
36
- data = attrs['data'].first
37
- timeout = int_decode(attrs['timeout'].first)
38
- yield id, created_at, data, timeout
39
- end
30
+ @domain.items.select('timeout', 'data', 'created_at',
31
+ :where => "created_at != '' AND timeout > '#{int_encode(0)}'",
32
+ :order => [:timeout, :asc],
33
+ :consistent_read => @consistent_read) {|itemdata|
34
+ id = itemdata.name
35
+ attrs = itemdata.attributes
36
+ created_at = int_decode(attrs['created_at'].first)
37
+ data = attrs['data'].first
38
+ timeout = int_decode(attrs['timeout'].first)
39
+ yield id, created_at, data, timeout
40
40
  }
41
41
  end
42
42
 
@@ -31,108 +31,108 @@ op.on('-v', '--verbose', "verbose mode", TrueClass) {|b|
31
31
  }
32
32
  op.separator("")
33
33
 
34
- op.on('--push ID=DATA', 'Push a task to the queue') {|s|
34
+ op.on('--push <ID> <DATA>', 'Push a task to the queue') {|s|
35
35
  type = :push
36
- id, data = s.split('=',2)
36
+ id = s
37
37
  }
38
38
 
39
39
  op.on('--list', 'Show queued tasks', TrueClass) {|b|
40
40
  type = :list
41
41
  }
42
42
 
43
- op.on('--cancel ID', 'Cancel a queued task') {|s|
43
+ op.on('--cancel <ID>', 'Cancel a queued task') {|s|
44
44
  type = :cancel
45
45
  id = s
46
46
  }
47
47
 
48
- op.on('--configure PATH.yaml', 'Write configuration file') {|s|
48
+ op.on('--configure <PATH.yaml>', 'Write configuration file') {|s|
49
49
  type = :conf
50
50
  confout = s
51
51
  }
52
52
 
53
53
  op.separator("")
54
54
 
55
- op.on('--exec COMMAND', 'Execute command') {|s|
55
+ op.on('--exec <COMMAND>', 'Execute command') {|s|
56
56
  type = :exec
57
57
  conf[:exec] = s
58
58
  }
59
59
 
60
- op.on('--run SCRIPT.rb', 'Run method named \'run\' defined in the script') {|s|
60
+ op.on('--run <SCRIPT.rb>', 'Run method named \'run\' defined in the script') {|s|
61
61
  type = :run
62
62
  conf[:run] = s
63
63
  }
64
64
 
65
65
  op.separator("")
66
66
 
67
- op.on('-f', '--file PATH.yaml', 'Read configuration file') {|s|
68
- conf[:file] = s
67
+ op.on('-f', '--file <PATH.yaml>', 'Read configuration file') {|s|
68
+ (conf[:files] ||= []) << s
69
69
  }
70
70
 
71
71
  op.on('-C', '--run-class', 'Class name for --run (default: ::Run)') {|s|
72
72
  conf[:run_class] = s
73
73
  }
74
74
 
75
- op.on('-t', '--timeout SEC', 'Time for another worker to take over a task when this worker goes down (default: 600)', Integer) {|i|
75
+ op.on('-t', '--timeout <SEC>', 'Time for another worker to take over a task when this worker goes down (default: 600)', Integer) {|i|
76
76
  conf[:timeout] = i
77
77
  }
78
78
 
79
- op.on('-b', '--heartbeat-interval SEC', 'Threshold time to extend the timeout (heartbeat interval) (default: timeout * 3/4)', Integer) {|i|
79
+ op.on('-b', '--heartbeat-interval <SEC>', 'Threshold time to extend the timeout (heartbeat interval) (default: timeout * 3/4)', Integer) {|i|
80
80
  conf[:heartbeat_interval] = i
81
81
  }
82
82
 
83
- op.on('-x', '--kill-timeout SEC', 'Threshold time to kill a task process (default: timeout * 10)', Integer) {|i|
83
+ op.on('-x', '--kill-timeout <SEC>', 'Threshold time to kill a task process (default: timeout * 10)', Integer) {|i|
84
84
  conf[:kill_timeout] = i
85
85
  }
86
86
 
87
- op.on('-X', '--kill-interval SEC', 'Threshold time to retry killing a task process (default: 60)', Integer) {|i|
87
+ op.on('-X', '--kill-interval <SEC>', 'Threshold time to retry killing a task process (default: 60)', Integer) {|i|
88
88
  conf[:kill_interval] = i
89
89
  }
90
90
 
91
- op.on('-i', '--poll-interval SEC', 'Polling interval (default: 1)', Integer) {|i|
91
+ op.on('-i', '--poll-interval <SEC>', 'Polling interval (default: 1)', Integer) {|i|
92
92
  conf[:poll_interval] = i
93
93
  }
94
94
 
95
- op.on('-r', '--retry-wait SEC', 'Time to retry a task when it is failed (default: same as timeout)', Integer) {|i|
95
+ op.on('-r', '--retry-wait <SEC>', 'Time to retry a task when it is failed (default: same as timeout)', Integer) {|i|
96
96
  conf[:retry_wait] = i
97
97
  }
98
98
 
99
- op.on('-e', '--expire SEC', 'Threshold time to expire a task (default: 345600 (4days))', Integer) {|i|
99
+ op.on('-e', '--expire <SEC>', 'Threshold time to expire a task (default: 345601 (4days))', Integer) {|i|
100
100
  conf[:expire] = i
101
101
  }
102
102
 
103
103
  op.separator("")
104
104
 
105
- op.on('--database URI', 'Use RDBMS for the backend database (e.g.: mysql://user:password@localhost/mydb)') {|s|
105
+ op.on('--database <URI>', 'Use RDBMS for the backend database (e.g.: mysql://user:password@localhost/mydb)') {|s|
106
106
  conf[:backend_database] = s
107
107
  }
108
108
 
109
- op.on('--table NAME', 'backend: name of the table (default: perfectqueue)') {|s|
109
+ op.on('--table <NAME>', 'backend: name of the table (default: perfectqueue)') {|s|
110
110
  conf[:backend_table] = s
111
111
  }
112
112
 
113
- op.on('--simpledb DOMAIN', 'Use Amazon SimpleDB for the backend database (e.g.: --simpledb mydomain -k KEY_ID -s SEC_KEY)') {|s|
113
+ op.on('--simpledb <DOMAIN>', 'Use Amazon SimpleDB for the backend database (e.g.: --simpledb mydomain -k KEY_ID -s SEC_KEY)') {|s|
114
114
  conf[:backend_simpledb] = s
115
115
  }
116
116
 
117
- op.on('-k', '--key-id ID', 'AWS Access Key ID') {|s|
117
+ op.on('-k', '--key-id <ID>', 'AWS Access Key ID') {|s|
118
118
  conf[:backend_key_id] = s
119
119
  }
120
120
 
121
- op.on('-s', '--secret-key KEY', 'AWS Secret Access Key') {|s|
121
+ op.on('-s', '--secret-key <KEY>', 'AWS Secret Access Key') {|s|
122
122
  conf[:backend_secret_key] = s
123
123
  }
124
124
 
125
125
  op.separator("")
126
126
 
127
- op.on('-w', '--worker NUM', 'Number of worker threads (default: 1)', Integer) {|i|
127
+ op.on('-w', '--worker <NUM>', 'Number of worker threads (default: 1)', Integer) {|i|
128
128
  conf[:workers] = i
129
129
  }
130
130
 
131
- op.on('-d', '--daemon PIDFILE', 'Daemonize (default: foreground)') {|s|
131
+ op.on('-d', '--daemon <PIDFILE>', 'Daemonize (default: foreground)') {|s|
132
132
  conf[:daemon] = s
133
133
  }
134
134
 
135
- op.on('-o', '--log PATH', "log file path") {|s|
135
+ op.on('-o', '--log <PATH>', "log file path") {|s|
136
136
  conf[:log] = s
137
137
  }
138
138
 
@@ -159,15 +159,29 @@ begin
159
159
  end
160
160
  op.parse!(argv)
161
161
 
162
- if argv.length != 0
163
- usage nil
162
+ case type
163
+ when :push
164
+ if argv.length != 1
165
+ usage nil
166
+ end
167
+ data = argv[0]
168
+
169
+ else
170
+ if argv.length != 0
171
+ usage nil
172
+ end
164
173
  end
165
174
 
166
- if conf[:file]
175
+ if conf[:files]
167
176
  require 'yaml'
168
- yaml = YAML.load File.read(conf[:file])
177
+ docs = ''
178
+ conf[:files].each {|file|
179
+ docs << File.read(file)
180
+ }
169
181
  y = {}
170
- yaml.each_pair {|k,v| y[k.to_sym] = v }
182
+ YAML.load_documents(docs) {|yaml|
183
+ yaml.each_pair {|k,v| y[k.to_sym] = v }
184
+ }
171
185
 
172
186
  conf = defaults.merge(y).merge(conf)
173
187
 
@@ -265,7 +279,7 @@ when :cancel
265
279
  if canceled
266
280
  puts "Task id=#{id} is canceled."
267
281
  else
268
- puts "Task id=#{id} does not exist."
282
+ puts "Task id=#{id} does not exist. abort"
269
283
  end
270
284
 
271
285
  when :push
@@ -273,7 +287,7 @@ when :push
273
287
  if submitted
274
288
  puts "Task id=#{id} is submitted."
275
289
  else
276
- puts "Task id=#{id} already exists."
290
+ puts "Task id=#{id} is duplicated. abort."
277
291
  end
278
292
 
279
293
  when :exec, :run
@@ -1,5 +1,5 @@
1
1
  module PerfectQueue
2
2
 
3
- VERSION = '0.7.0'
3
+ VERSION = '0.7.1'
4
4
 
5
5
  end
@@ -37,28 +37,32 @@ class MonitorThread
37
37
  @cond.wait(@mutex)
38
38
  end
39
39
  }
40
- while true
41
- sleep 1
42
- @mutex.synchronize {
43
- return if @engine.finished?
44
- break unless @token
45
- now = Time.now.to_i
46
- try_extend(now)
47
- try_kill(now)
48
- }
49
- end
40
+ process
50
41
  end
51
42
  rescue
52
43
  @engine.stop($!)
53
44
  end
54
45
 
46
+ def process
47
+ while true
48
+ sleep 1
49
+ @mutex.synchronize {
50
+ return if @engine.finished?
51
+ return unless @token
52
+ now = Time.now.to_i
53
+ try_extend(now)
54
+ try_kill(now)
55
+ }
56
+ end
57
+ end
58
+
55
59
  def try_extend(now)
56
60
  if now >= @heartbeat_time && !@canceled
57
- @log.debug "extending timeout=#{now+@timeout} id=#{@task.id}"
61
+ @log.debug "extending timeout=#{now+@timeout} id=#{@task_id}"
58
62
  begin
59
63
  @backend.update(@token, now+@timeout)
60
64
  rescue CanceledError
61
- @log.info "task id=#{@task.id} is canceled."
65
+ @log.info "task id=#{@task_id} is canceled."
62
66
  @canceled = true
63
67
  @kill_time = now
64
68
  end
@@ -75,8 +79,15 @@ class MonitorThread
75
79
 
76
80
  def kill!
77
81
  if @kill_proc
78
- @log.info "killing #{@task.id}..."
79
- @kill_proc.call rescue nil
82
+ @log.info "killing id=#{@task_id}..."
83
+ begin
84
+ @kill_proc.call
85
+ rescue
86
+ @log.info "kill failed id=#{@task_id}: #{$!}"
87
+ $!.backtrace.each {|bt|
88
+ $log.debug " #{bt}"
89
+ }
90
+ end
80
91
  end
81
92
  end
82
93
 
@@ -90,10 +101,11 @@ class MonitorThread
90
101
  @thread.join
91
102
  end
92
103
 
93
- def set(token)
104
+ def set(token, task_id)
94
105
  @mutex.synchronize {
95
106
  now = Time.now.to_i
96
107
  @token = token
108
+ @task_id = task_id
97
109
  @heartbeat_time = now + @heartbeat_interval
98
110
  @kill_time = now + @kill_timeout
99
111
  @kill_proc = nil
@@ -165,7 +177,7 @@ class Worker
165
177
  def process(token, task)
166
178
  @log.info "processing task id=#{task.id}"
167
179
 
168
- @monitor.set(token)
180
+ @monitor.set(token, task.id)
169
181
  success = false
170
182
  begin
171
183
  run = @run_class.new(task)
@@ -181,6 +193,9 @@ class Worker
181
193
 
182
194
  rescue
183
195
  @log.info "failed id=#{task.id}: #{$!}"
196
+ $!.backtrace.each {|bt|
197
+ $log.debug " #{bt}"
198
+ }
184
199
 
185
200
  ensure
186
201
  @monitor.reset(success)
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: perfectqueue
3
3
  version: !ruby/object:Gem::Version
4
- hash: 3
4
+ hash: 1
5
5
  prerelease: false
6
6
  segments:
7
7
  - 0
8
8
  - 7
9
- - 0
10
- version: 0.7.0
9
+ - 1
10
+ version: 0.7.1
11
11
  platform: ruby
12
12
  authors:
13
13
  - Sadayuki Furuhashi