require 'debci'
require 'debci/user'
require 'debci/amqp'
require 'debci/backend'
require 'debci/db'
require 'debci/package'
require 'debci/package_status'
require 'debci/test/duration'
require 'debci/test/paths'
require 'cgi'
require 'time'
require 'pathname'
require 'debci/validators'
require 'debci/worker'
module Debci
class Job < ActiveRecord::Base
PUBLIC_API = {
trigger: 'the same string that was provided in the test submission. (string)',
package: 'name of tested package (string)',
arch: 'architecture where the test ran (string)',
suite: 'suite where the test ran (string)',
version: 'version of the package that was tested (string)',
status: '"pass", "fail", or "tmpfail" (string), or *null* if the test didn\'t finish yet.',
run_id: 'an id for the test run, generated by debci (integer)',
is_private: 'bool value to specify whether test is private or not',
extra_apt_sources: 'an array specifying extra apt sources added to `/etc/apt/sources.list.d` for the test.',
updated_at: 'timestamp of the last update to the job (string)',
date: 'timestamp of when the test run finished (string)',
}.freeze
def public_api_attributes
data = PUBLIC_API.keys.each_with_object({}) do |k, memo|
memo[k] = self[k]
end
data["package"] = package.name
data
end
include Debci::Validators::APTSource
belongs_to :package, class_name: 'Debci::Package'
belongs_to :requestor, class_name: 'Debci::User', foreign_key: 'requestor_id'
belongs_to :worker, class_name: 'Debci::Worker', foreign_key: 'worker_id'
validates :requestor, presence: true
has_many :package_status, class_name: 'Debci::PackageStatus'
scope :newsworthy, -> { not_pinned.where(['status in (?) AND previous_status in (?) and status != previous_status', ['pass', 'fail', 'neutral'], ['pass', 'fail', 'neutral']]) }
scope :finished, -> { where('status is NOT NULL') }
scope :not_pinned, -> { where('pin_packages is NULL') }
scope :not_private, -> { where(is_private: false) }
def pinned?
!pin_packages.empty?
end
scope :status_on, lambda { |suite, arch|
not_private.joins(:package_status, :package).where(packages: { removed: false }).where(['package_statuses.suite IN (?) AND package_statuses.arch IN (?)', suite, arch])
}
scope :all_status, lambda {
status_on(
Debci.config.suite_list,
Debci.config.arch_list
)
}
scope :tmpfail, -> { all_status.where(status: 'tmpfail') }
scope :fail, -> { all_status.where(status: 'fail') }
scope :visible, lambda {
last_visible_time = Time.now - Debci.config.status_visible_days.days
where('date > :time', time: last_visible_time)
}
scope :slow, lambda {
all_status.where('duration_seconds > :time', time: Debci.config.slow_tests_duration_minutes.minutes)
}
after_save do |job|
next if job.is_private
next unless job.status
next unless job.date
next if job.pinned?
next if job.history.where(['date > ?', date]).exists?
job.transaction do
status = Debci::PackageStatus.find_or_initialize_by(
package: self.package,
suite: self.suite,
arch: self.arch,
)
status.job = job
status.save!
end
end
def self.platform_specific_issues
all_status.includes(:package).group_by(&:package).select do |_, statuses|
statuses.map(&:status).uniq.size > 1
end
end
include Debci::Test::Duration
include Debci::Test::Paths
serialize :pin_packages, Array
serialize :extra_apt_sources, Array
class InvalidStatusFile < RuntimeError; end
def self.import(status_file)
status = JSON.parse(File.read(status_file))
run_id = status.delete('run_id').to_i
package = status.delete('package')
job = Debci::Job.find(run_id)
if package != job.package.name
raise InvalidStatusFile.new("Data in %{file} is for package %{pkg}, while database says that job %{id} is for package %{origpkg}" % {
file: status_file,
pkg: package,
id: run_id,
origpkg: job.package,
})
end
status.each do |k, v|
job.send("#{k}=", v)
end
job.save!
job
end
def self.receive(directory)
src = Pathname(directory)
id = src.basename.to_s
begin
job = Debci::Job.find(id)
rescue ActiveRecord::RecordNotFound
nil
else
exitcode = src / 'exitcode'
unless exitcode.exist?
job.status = "tmpfail"
job.message = "Invalid test results received"
return job
end
job.status, job.message = status(exitcode.read.to_i)
duration = (src / 'duration')
if duration.exist?
job.duration_seconds = duration.read.to_i
job.date = duration.stat.mtime
else
job.duration_seconds = 0
job.date = Time.now
end
worker_file = (src / 'worker')
if worker_file.exist?
job.worker = Debci::Worker.find_or_create_by!(name: worker_file.read.strip)
end
testpkg_version = src / 'testpkg-version'
if testpkg_version.exist?
job.version = testpkg_version.read.split.last if testpkg_version
else
job.version = 'n/a'
end
if job.previous
job.previous_status = job.previous.status
end
if job.last_pass
job.last_pass_date = job.last_pass.date
job.last_pass_version = job.last_pass.version
end
dest = job.autopkgtest_data_dir
dest.parent.mkpath
dest.rmtree if dest.exist?
FileUtils.cp_r src, dest
Dir.chdir dest do
artifacts = Dir['*'] - ['log.gz']
cmd = ['tar', '-caf', 'artifacts.tar.gz', '--remove-files', '--', *artifacts]
system(*cmd) || raise('Command failed: %<cmd>s' % { cmd: cmd.join(' ') })
end
job.calculate_file_sizes!
job.save!
src.rmtree
job
end
end
def autopkgtest_data_path
@autopkgtest_data_path ||= File.join(suite, arch, package.prefix, package.name, id.to_s)
end
def autopkgtest_data_dir
@autopkgtest_data_dir ||= Pathname(Debci.config.autopkgtest_basedir) / autopkgtest_data_path
end
attribute :files_purged, :boolean, default: false
def cleanup(reason: "no reason given")
self.purge_files
self.files_purged = true
self.save!
Debci.log(
'Cleaned up files for job %<job_id>s (%<job>s): %<reason>s' % {
job_id: self.run_id,
job: self,
reason: reason,
}
)
end
def calculate_file_sizes!
dir = self.autopkgtest_data_dir
log = (dir / 'log.gz')
self.log_size = log.size if log.exist?
self.artifacts_size = (dir / 'artifacts.tar.gz').size
end
def disk_usage
(self.log_size || 0) + (self.artifacts_size || 0)
end
def self.status(exit_code)
case exit_code
when 0
['pass', 'All tests passed']
when 2
['pass', 'Tests passed, but at least one test skipped']
when 4
['fail', 'Tests failed']
when 6
['fail', 'Tests failed, and at least one test skipped']
when 12, 14
['fail', 'Erroneous package']
when 8
['neutral', 'No tests in this package or all skipped']
when 16
['tmpfail', 'Could not run tests due to a temporary testbed failure']
else
['tmpfail', "Unexpected autopkgtest exit code #{exit_code}"]
end
end
def self.pending
Debci::Job.includes(:requestor).not_private.where(status: nil).order(:created_at)
end
def self.history(package, suite, arch)
Debci::Job.includes(:requestor).not_private.finished.where(
package: package,
suite: suite,
arch: arch
).order('date')
end
def history
@history ||= self.class.history(package, suite, arch)
end
def previous_unpinned_jobs
@previous_unpinned_jobs ||= history.not_pinned.where(["date < ?", date])
end
def previous
@previous ||= previous_unpinned_jobs.last
end
def last_pass
@last_pass ||= previous_unpinned_jobs.where(status: 'pass').last
end
def time
days = (Time.now - self.created_at)/86400
if days >= 1 || days <= -1
"#{days.floor} day(s) ago"
else
"#{Time.at(Time.now - self.created_at).gmtime.strftime('%H')} hour(s) ago"
end
end
def as_json(options = nil)
super(options).update(
"duration_human" => self.duration_human,
"package" => package.name,
)
end
def enqueue_parameters
parameters = ['run-id:%s' % id]
if self.trigger
parameters << "trigger:#{CGI.escape(trigger)}"
end
Array(self.pin_packages).each do |pin|
*pkgs, suite = pin
parameters << "pin-packages:#{suite}=#{pkgs.join(',')}"
end
Array(self.).each do |name|
src = Debci..find(name)
parameters << "extra-apt-source:#{Base64.strict_encode64(src.entry)}"
parameters << "signing-key:#{Base64.strict_encode64(src.signing_key)}" if src.signing_key
end
parameters
end
def enqueue(priority = 5)
backend = Debci::Backend.select(package, arch)
self.update(requested_backend: backend)
queue = Debci::AMQP.get_queue(arch, backend)
parameters = enqueue_parameters
queue.publish("%s %s %s" % [package.name, suite, parameters.join(' ')], priority: priority)
end
def backend
requested_backend || Debci::Backend.default
end
def retry
new_job = Debci::Job.create!(
package: self.package,
suite: self.suite,
arch: self.arch,
requestor: self.requestor,
trigger: self.trigger,
pin_packages: self.pin_packages,
is_private: self.is_private,
extra_apt_sources: self.
)
new_job.enqueue
new_job
end
def to_s
"%s %s/%s (%s)" % [package.name, suite, arch, status || 'pending']
end
def title
'%s %s' % [version, status]
end
def headline
"#{package.name} #{version} #{status.upcase} on #{suite}/#{arch}"
end
def always_failing?
last_pass_version.nil? || last_pass_version == 'n/a'
end
def had_success?
!always_failing?
end
end
end