require 'digest/sha1'
require 'fileutils'
require 'json'
require 'rdoc'
require 'securerandom'
require "sinatra/namespace"
require 'time'
require 'debci'
require 'debci/app'
require 'debci/job'
require 'debci/user'
require 'debci/test_handler'
require 'debci/validators'
class SelfDocAPI < Debci::App
get '/doc' do
@doc = self.class.doc
erb :doc
end
class << self
def doc(d=nil)
@last_doc = format_doc(d)
@doc ||= []
end
def register_doc(method, path)
@last_doc ||= nil
return unless @last_doc
entry = {
method: method,
path: path,
text: @last_doc,
anchor: [method, path].join('_'),
}
@last_doc = nil
doc.push(entry)
end
def format_doc(text)
return nil unless text
lines = text.lines
if lines.first && lines.first.strip == ""
lines.first.shift
end
lines.first =~ /^\s+/; prefix = ::Regexp.last_match(0)
if prefix
lines.map! do |line|
line.sub(/^#{prefix}/, '')
end
end
formatter = RDoc::Markup::ToHtml.new(RDoc::Options.new, nil)
RDoc::Markdown.parse(lines.join).accept(formatter)
end
def get(path, *args)
register_doc('GET', path)
super(path, *args)
end
def post(path, *args)
register_doc('POST', path)
super(path, *args)
end
end
end
module Debci
class API < SelfDocAPI
include Debci::TestHandler
include Debci::Validators::APTSource
register Sinatra::Namespace
set :views, "#{File.dirname(__FILE__)}/api"
attr_reader :suite, :arch, :user
get '/' do
redirect "#{request.script_name}/doc"
end
namespace '/v1' do
doc <<-EOF
* bli
* bli
* bli
EOF
doc <<-EOF
This endpoint can be used to test your API key. It returns a 200 (OK)
HTTP status if your API key is valid, and 403 (Forbidden) otherwise. The
response also includes the username corresponding to the API key in the
`Auth-User` header.
EOF
get '/auth' do
authenticate_key!
200
end
doc <<-EOF
Gets the reject list i.e. list of packages which won't be tested by this
DebCI instance.
The response is a JSON object where each key is a package. The value of
each package is an object where each key is a suite for which it is
blocked. Each suite key has an object with blocked architecture as key
which is followed by a level of versions. The wildcard `*` is a valid
value signifying all suites.
```
{
"package": {
"suite": {
"arch": {
"version": ""
}
}
}
}
```
Example:
```
{
"bash": {
"testing": {
"*": {
"*": ""
}
},
"unstable": {
"amd64": {
"5.0.0": ""
}
}
}
}
```
This blocks `bash` from `testing` for all architectures and versions. It
also blocks it for unstable but only for amd64 and version 5.0.0.
EOF
get '/reject_list' do
authenticate_key!
content_type :json
Debci.reject_list.data.to_json
end
doc <<-EOF
Presents a simple UI for retrying a test
EOF
get '/retry/:run_id' do
redirect "user/:user/retry/#{params[:run_id]}"
end
doc <<-EOF
Retries the test with the given ID.
EOF
post '/retry/:run_id' do
job = get_job_to_retry(params[:run_id])
job.retry
200
end
test_fields = Debci::Job::PUBLIC_API
doc <<-EOF
Retrieves results for your test requests.
Parameters:
* `since`: UNIX timestamp; tells the API to only retrieve results that are
newer then the given timestamp, i.e. jobs that were either created after
it, or finished after it.
Some test results may be returned in multiple calls to this endpoint that
use different value of `since`. For example, while a test is still running,
it will be returned, but its status will be `null`. After it is
completed, it will be updated to have the final status. So, if you are
processing test results, make sure you support receiving the same result
more than once, and updating the corresponding data on your side.
The response is a JSON object containing the following keys:
* `until`: UNIX timestamp that represents the timestamp of the latest results
available. can be used as the `since` parameter in subsequent requests to
limit the list of results to only the ones newer than it.
* `results`: a list of test results, each of each will containing at least the
following items:
#{test_fields.map { |f, desc| "\n * `#{f}`: #{desc}" }.join}
Note that this endpoint will only list requests that were made by the same API
key that is being used to call it.
Example:
```
$ curl --header "Auth-Key: $KEY" https://host/api/v1/test?since=1508072999
{
"until": 1508159423,
"results": [
{
"trigger": "foo/1.2",
"package": "bar",
"arch": "amd64",
"suite": "testing",
"version": "4.5",
"status": "fail",
"run_id": 12345
"is_private": false,
"extra_apt_sources": [],
},
{
"trigger": "foo/1.2",
"package": "baz",
"arch": "amd64",
"suite": "testing",
"version": "2.7",
"status": "pass",
"run_id": 12346,
"is_private": true,
"extra_apt_sources": [],
}
]
}
```
EOF
get '/test' do
authenticate_key!
jobs = Debci::Job.where(requestor: @user)
jobs = jobs.includes([:package])
jobs = jobs.order('created_at')
if params[:since]
since = Time.strptime(params[:since], '%s')
jobs = jobs.where('created_at >= ? OR date >= ?', since, since)
end
results = []
fields = test_fields.keys.map(&:to_s)
jobs.in_batches do |batch|
batch.each do |job|
results << job.public_api_attributes
end
end
data = {
until: jobs.map(&:created_at).max.to_i,
results: results,
}
['Content-Type'] = 'application/json'
data.to_json
end
before '/test/:suite/:arch*' do
authenticate_key!
@suite = params[:suite]
@arch = params[:arch]
@priority = params.fetch("priority", 5).to_i
if !Debci.config.arch_list.include?(arch)
halt(400, "Invalid architecture: #{arch}\n")
elsif !Debci.config.suite_list.include?(suite)
halt(400, "Invalid suite: #{suite}\n")
elsif !validate_priority(@priority)
halt(400, "Invalid priority: #{@priority}\n")
end
end
doc <<-EOF
```
EOF
post '/test/batch' do
test_requests = load_json(params[:tests])
errors = validate_batch_test(test_requests)
if errors.empty?
request_batch_tests(test_requests, @user, @priority)
201
else
halt(400, "Error: #{errors.join("\n")}")
end
end
doc <<-EOF
URL parameters:
* `:suite`: which suite to test
* `:arch`: which architecture to test
Other parameters:
* `tests`: a JSON object describing the tests to be executed. This parameter can
be either a file upload or a regular POST parameter.
* `priority`: an integer between 1 and 10 describing the priority to assign the
requested tests
The `tests` JSON object must be an *Array* of objects. Each object represents a
single test request, and can contain the following keys:
* `package`: the (source!) package to be tested
* `trigger`: a string that identifies the reason why this test is being
requested. debci only stores this string, and it does not handle this in any
special way.
* `pin-packages`: an array describing packages that need to be obtained from
different suites than the main one specified by the `suite` parameter. This
is used e.g. to run tests on `testing` with a few packages from `unstable`,
or on `unstable` with a few packages from `experimental`. Each item of the
array is another array with 2 elements: the first is the package, and the
second is the source. Examples:
* `["foo", "unstable"]`: get `foo` from unstable
* `["src:bar", "unstable"]`: get all binaries built from `bar` from unstable
* `["foo,src:bar", "unstable"]`: get `foo` and all binaries built from `bar` from unstable
Note: each suite can be only present once.
* `is_private`: bool value that need to be set `true` if test is private and if not
given takes `false` as default.
* `extra-apt-sources`: an array specifying extra apt sources which is added to
`/etc/apt/sources.list.d`. Note that if you use both `extra-apt-sources` and
`pin-packages`, then you must pass all APT sources necessary to resolve your
pinnings.
In the example below, we are requesting to test `debci` and `vagrant` from
testing, but with all binaries that come from the `ruby-defaults` source coming
from unstable:
```
$ cat tests.json
[
{
"package": "debci",
"trigger": "ruby/X.Y",
"pin-packages": [
["src:ruby-defaults", "unstable"]
],
"is_private": true,
"extra-apt-sources": [
"bullseye-security"
]
},
{
"package": "vagrant",
"trigger": "ruby/X.Y",
"pin-packages": [
["src:ruby-defaults", "unstable"]
]
}
]
# tests as a file upload
$ curl --header "Auth-Key: $KEY" --form tests=@tests.json \
https://host/api/v1/test/testing/amd64
# tests as a regular POST parameter, with specific priority
$ curl --header "Auth-Key: $KEY" --data tests="$(cat tests.json)" --data priority=8 \
https://host/api/v1/test/testing/amd64
```
EOF
post '/test/:suite/:arch' do
tests = load_json(params[:tests])
errors = validate_tests(tests)
if errors.empty?
self.request_tests(tests, suite, arch, @user, @priority)
201
else
halt(400, "Invalid request: #{errors.join("\n")}")
end
end
doc <<-EOF
This is a shortcut to request a test run for a single package.
URL parameters:
* `:suite`: which suite to test
* `:arch`: which architecture to test
* `:package`: which (source!) package to test
* `priority`: an integer between 1 and 10 describing the priority to assign the
requested tests
* `is_private`: bool value that need to be set `true` if test is private and if not
given takes `false` as default
* `extra_apt_sources`: a JSON array specifying extra apt sources which is added to
`/etc/apt/sources.list.d`.
Example:
```
$ curl --header "Auth-Key: $KEY" --data '' https://host/api/v1/test/unstable/amd64/debci
```
EOF
post '/test/:suite/:arch/:package' do
pkg = params[:package]
if Debci.reject_list.include?(pkg, suite: suite, arch: arch)
halt(400, "RejectListed package: #{pkg}\n")
elsif ! valid_package_name?(pkg)
halt(400, "Invalid package name: #{pkg}\n")
elsif ! params[:is_private].in? ["true", "false", nil]
halt(400, "Invalid value for bool parameter is_private for package: #{pkg}\n")
end
if params[:extra_apt_sources]
begin
= JSON.parse(params[:extra_apt_sources])
= ()
halt(400, "Invalid extra apt sources: #{}") unless .empty?
rescue JSON::ParserError => error
halt(400, "Invalid JSON: #{error}")
end
end
package = Debci::Package.find_or_create_by!(name: pkg)
job = Debci::Job.create!(
package: package,
suite: params[:suite],
arch: params[:arch],
requestor: @user,
is_private: params[:is_private] == "true",
extra_apt_sources:
)
self.enqueue(job, @priority)
201
end
doc <<-EOF
To publish private tests.
Parameter:
`run_ids`: a string of run ids of private tests to be published separated by `comma (,)`
Example:
```
curl --header "Auth-Key: $KEY" --data 'run_ids=25,26' https://host/api/v1/test/publish
```
EOF
post '/test/publish' do
authenticate_key!
run_ids = (params[:run_ids] || '').split(/\s*,\s*/).grep(/^\d+$/).map(&:to_i)
jobs = Debci::Job.where(requestor: @user, run_id: run_ids)
jobs.update_all(is_private: false)
200
end
end
protected
def load_json(param)
begin
raise "No tests" if param.nil?
str = (param.is_a?(Hash) && File.read(param[:tempfile])) || param
JSON.parse(str)
rescue JSON::ParserError => error
halt(400, "Invalid JSON: #{error}")
rescue StandardError => error
halt(400, "Error: #{error}")
end
end
def authenticate_key!
key = env['HTTP_AUTH_KEY']
if key && @user = Debci::Key.authenticate(key)
response['Auth-User'] = @user.username
else
halt(403, "Invalid key\n")
end
end
end
end