Skip to content

Commit

Permalink
B #6171: Improve LinuxContainers Marketplace queries (#2603)
Browse files Browse the repository at this point in the history 
Instead of traversing links in https://images.linuxcontainers.org/ (which takes
a lot of time), now the LXC image information is extracted parsing the file
https://images.linuxcontainers.org/streams/v1/images.json

(cherry picked from commit 5748f96c17982482d98f041ebe3d8ea88ed33ee4)
(cherry picked from commit e4343eb05171285f4780b9269d96ac2c411820a5)
  • Loading branch information
@brodriguez-opennebula @rsmontero
brodriguez-opennebula authored and rsmontero committed May 17, 2023
1 parent 420cd10 commit 6d09265
Showing 1 changed file with 93 additions and 108 deletions.
201 changes: 93 additions & 108 deletions src/market_mad/remotes/linuxcontainers/monitor
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,18 +44,18 @@ class LinuxContainersMarket
# :privileged container idmap
#---------------------------------------------------------------------------
DEFAULTS = {
:url => 'https://images.linuxcontainers.org',
:sizemb => 2048,
:fs => 'ext4',
:format => 'raw',
:agent => 'OpenNebula',
:tested_apps => ['alpine', 'alt', 'centos', 'rockylinux', 'almalinux', 'oracle', 'debian',
'ubuntu', 'fedora', 'devuan', 'opensuse', 'amazonlinux'],
:url => 'https://images.linuxcontainers.org/',
:sizemb => 2048,
:fs => 'ext4',
:format => 'raw',
:agent => 'OpenNebula',
:tested_apps => %w[alpine alt centos rockylinux almalinux oracle debian
ubuntu fedora devuan opensuse amazonlinux],
:skip_untested => 'yes',
:memory => '768',
:cpu => 1,
:vcpu => 2,
:privileged => true
:memory => '768',
:cpu => 1,
:vcpu => 2,
:privileged => true
}

def initialize(options = {})
Expand All @@ -64,7 +64,7 @@ class LinuxContainersMarket

version_path = File.dirname(__FILE__) + '/../../VERSION'
@options[:agent] = "OpenNebula #{File.read(version_path)}" \
if File.exist? version_path
if File.exist? version_path

@http_client = nil
end
Expand All @@ -83,100 +83,93 @@ class LinuxContainersMarket
[response.code.to_i, response.msg]
end

# Get the list of appliances
# Relevant image.json format fields are
#
# {...,
# "products": {
# $distro:$version:$arch:$variant: {
# ...
# "versions": {
# $release_date: {
# "items": {
# $container_type: {
# ...
# "size": int
# "path": $path
# "sha256": $sha256
# }
# }
# }
# }
# }
# }
# }

def fetch_appliances
first_level = '/images/'
image_list_address = '/streams/v1/images.json'

open_connection

rc, body = get(first_level)
rc, body = get(image_list_address)

return rc, body if rc != 0

distros = body.scan(%r{a href="([a-z].*/)">})
distros.map! {|d| d.join.chomp('/') }

if @options[:skip_untested] == 'yes'
distros.delete_if {|d| !@options[:tested_apps].include?(d) }
end

tree = {}

distros.each do |distro|
d_suf = "#{distro}/"
rc, body = get(first_level + d_suf)

next if rc != 0

versions = body.scan(%r{a href="(.*/)">})
versions.shift # Remove first entry ("Parent Directory")
image_list = JSON.parse(body)
appstr = ''

version_path = {}
versions.each do |version|
path = "#{first_level}#{d_suf}#{version[0]}amd64/default/"
rc, body = get(path)
image_list['products'].each do |img, img_data|
distro, version, arch, variant = img.split(':')

next if rc != 0
next if variant != 'default' || arch != 'amd64'
next if @options[:skip_untested] == 'yes' && !@options[:tested_apps].include?(distro)

release_dates = body.scan(%r{a href="(.*/)">})
# Use the last version for each container
regtime = img_data['versions'].to_a.last[0]

# Previous release_dates array leaves a point in the html page
release_date = release_dates.last[0]
version_path[version[0]] = "#{path}#{release_date}rootfs.tar.xz"
begin
path = img_data['versions'][regtime]['items']['root.tar.xz']['path']
rescue StandardError
next
end

tree[d_suf] = version_path
end

appstr = ''

#-----------------------------------------------------------------------
# Generate the container app information
#-----------------------------------------------------------------------
tree.each do |distro, value|
value.each do |version, path|
path = URI.decode_www_form_component(path)

description = "Downloaded from #{@options[:url]}"
regtime = app_time(path).to_s

data = {
'NAME' => "#{distro[0...-1]}_#{version[0...-1]} - LXD",
'SIZE' => @options[:sizemb],
'PUBLISHER' => @options[:url],
'FORMAT' => @options[:format],
'REGTIME' => regtime,
'SOURCE' => app_url(path).to_s,
'MD5' => md5(regtime),
'IMPORT_ID' => '-1',
'ORIGIN_ID' => '-1',
'VERSION' => '1.0',
'TYPE' => 'IMAGE',
'DESCRIPTION' => description,
'TAGS' => '',
'LINK' => "#{@options[:url]}#{path}"
.gsub('/rootfs.tar.xz', '')
}

tmpl = ''
data.each {|key, val| print_var(tmpl, key, val) }

tmpl64 = ''
print_var(tmpl64, 'DRIVER', 'raw')

data = { 'APPTEMPLATE64' => tmpl64,
'VMTEMPLATE64' => LXDMarket.template(@options) }
data.each do |key, val|
print_var(tmpl, key, Base64.strict_encode64(val))
end

appstr << "APP=\"#{Base64.strict_encode64(tmpl)}\"\n"
builddate = DateTime.strptime(regtime, '%Y%m%d_%H:%M').to_time.to_i

data = {
'NAME' => "#{distro}_#{version} - LXD",
'SIZE' => @options[:sizemb],
'PUBLISHER' => @options[:url],
'FORMAT' => @options[:format],
'REGTIME' => builddate,
'SOURCE' => app_url(path).to_s,
'MD5' => md5(regtime),
'IMPORT_ID' => '-1',
'ORIGIN_ID' => '-1',
'VERSION' => '1.0',
'TYPE' => 'IMAGE',
'DESCRIPTION' => "Downloaded from #{@options[:url]}",
'TAGS' => '',
'LINK' => "#{@options[:url]}#{path}"
}

tmpl = ''
data.each { |key, val| print_var(tmpl, key, val) }

tmpl64 = ''
print_var(tmpl64, 'DRIVER', 'raw')

data = { 'APPTEMPLATE64' => tmpl64,
'VMTEMPLATE64' => LXDMarket.template(@options) }

data.each do |key, val|
print_var(tmpl, key, Base64.strict_encode64(val))
end

appstr << "APP=\"#{Base64.strict_encode64(tmpl)}\"\n"
end

appstr

ensure
@http_client.finish if @http_client
@http_client&.finish if @http_client
end

private
Expand All @@ -190,14 +183,6 @@ class LinuxContainersMarket
"&filesystem=#{@options[:fs]}&format=#{@options[:format]}"
end

# Returns build date based on image path
def app_time(path)
buildate = path.split('/')[6]

buildate = DateTime.strptime(buildate, '%Y%m%d_%H:%M')
buildate.to_time.to_i
end

# Print variable in an APP template
def print_var(str, name, val)
return if val.nil?
Expand All @@ -213,7 +198,7 @@ class LinuxContainersMarket

# Opens a TCP connection to @options[:url]
def open_connection
@http_client.finish if @http_client
@http_client&.finish if @http_client

http_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY'] # for ruby 1.9.3

Expand Down Expand Up @@ -244,7 +229,7 @@ class LinuxContainersMarket
end

################################################################################
# Main Program. Outpust the list of marketplace appliances
# Main Program. Its output is the list of marketplace appliances
################################################################################
def set_option(option, doc, name, path)
option[name] = doc.elements[path].text if doc.elements[path]
Expand All @@ -258,18 +243,18 @@ begin
pre = 'MARKETPLACE/TEMPLATE'

data = {
:url => "#{pre}/ENDPOINT",
:sizemb => "#{pre}/IMAGE_SIZE_MB",
:fs => "#{pre}/FILESYSTEM",
:format => "#{pre}/FORMAT",
:url => "#{pre}/ENDPOINT",
:sizemb => "#{pre}/IMAGE_SIZE_MB",
:fs => "#{pre}/FILESYSTEM",
:format => "#{pre}/FORMAT",
:skip_untested => "#{pre}/SKIP_UNTESTED",
:memory => "#{pre}/MEMORY",
:cpu => "#{pre}/CPU",
:vcpu => "#{pre}/VCPU",
:privileged => "#{pre}/PRIVILEGED"
:memory => "#{pre}/MEMORY",
:cpu => "#{pre}/CPU",
:vcpu => "#{pre}/VCPU",
:privileged => "#{pre}/PRIVILEGED"
}

data.each {|key, value| set_option(options, doc, key, value) }
data.each { |key, value| set_option(options, doc, key, value) }

str = LinuxContainersMarket.new(options).fetch_appliances
puts str
Expand Down

0 comments on commit 6d09265

Please sign in to comment.