Skip to content

Commit 8833b70

Browse files
jmrootjmroot
committed
mpbb list-subports: check for archives from Tcl
This has several performance benefits: * MacPorts only has to be started up once * Each portfile only has to be parsed at most once * libcurl can keep the connection open and reuse it
1 parent a97bf2d commit 8833b70

File tree

4 files changed

+73
-50
lines changed

4 files changed

+73
-50
lines changed

‎README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,9 @@ Subcommand scripts may use but not modify these global shell parameters:
9494
- `$option_failcache_dir`:
9595
A directory for storing information about previously failed builds which
9696
saves time because builds that are known to fail will not be attempted.
97+
- `$option_license_db_dir`:
98+
A directory for storing information about port licenses used by the
99+
port_binary_distributable.tcl script.
97100

98101

99102
## Runtime Requirements ##

‎mpbb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ parseopt prefix:,work-dir: "$@" || exit
5858
option_log_dir=${option_work_dir}/logs
5959
}
6060
option_failcache_dir=${option_work_dir}/failcache
61+
option_license_db_dir=${option_work_dir}/license_db
6162

6263
# Inform the user if old repositories are still present.
6364
if [[ -d ${option_work_dir}/tools/.svn ]]; then

‎mpbb-list-subports

Lines changed: 3 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -31,60 +31,15 @@ print-subports() {
3131
local archive_site_public=$1
3232
local archive_site_private=$2
3333
local portnames=$3
34-
local port
35-
local ports
36-
local exclude
37-
local exclude_reasons
38-
local reason
3934

4035
# $option_prefix is set in mpbb
4136
# shellcheck disable=SC2154
4237
tclsh=${option_prefix}/bin/port-tclsh
4338
# $option_prefix is set in mpbb
4439
# shellcheck disable=SC2154
45-
ports=$("${tclsh}" "${thisdir}/tools/sort-with-subports.tcl" ${portnames}) || return
46-
for port in $ports; do
47-
exclude=0
48-
exclude_reasons=()
49-
50-
if [[ -n "${archive_site_public}" || -n "${archive_site_private}" ]]; then
51-
# FIXME: this doesn't take selected variants into account
52-
# $thisdir is set in mpbb
53-
# shellcheck disable=SC2154
54-
archive_path=$("${tclsh}" "${thisdir}/tools/archive-path.tcl" "${port}")
55-
archive_basename=$(basename "${archive_path}")
56-
57-
# $option_jobs_dir is set in mpbb
58-
# shellcheck disable=SC2154
59-
if "${tclsh}" "${option_jobs_dir}/port_binary_distributable.tcl" "${port}"; then
60-
archive_type=public
61-
archive_distributable="distributable"
62-
archive_site="${archive_site_public}"
63-
else
64-
archive_type=private
65-
archive_distributable="not distributable"
66-
archive_site="${archive_site_private}"
67-
fi
68-
69-
if [[ -n "${archive_site}" ]] && curl -fIsL "${archive_site}/${port}/${archive_basename}" > /dev/null; then
70-
exclude=1
71-
exclude_reasons+=("it is ${archive_distributable} and has already been built and uploaded to the ${archive_type} server")
72-
fi
73-
fi
74-
75-
if [ $exclude -eq 0 ]; then
76-
echo "$port"
77-
else
78-
if [ ${#exclude_reasons[@]} -eq 1 ]; then
79-
echo >&2 "Excluding '${port}' because ${exclude_reasons[0]}."
80-
else
81-
echo >&2 "Excluding '${port}' for the following reasons:"
82-
for reason in "${exclude_reasons[@]}"; do
83-
echo >&2 " - ${reason}"
84-
done
85-
fi
86-
fi
87-
done
40+
"${tclsh}" "${thisdir}/tools/sort-with-subports.tcl" --jobs_dir "${option_jobs_dir}" \
41+
--license_db_dir "${option_license_db_dir}" --archive_site_public "${archive_site_public}" \
42+
--archive_site_private "${archive_site_private}" ${portnames} || return
8843
}
8944

9045
list-subports() {

‎tools/sort-with-subports.tcl

Lines changed: 66 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# sub-ports of the specified ports.
66
#
77
# Copyright (c) 2006,2008 Bryan L Blackburn. All rights reserved.
8-
# Copyright (c) 2018 The MacPorts Project
8+
# Copyright (c) 2018-2019 The MacPorts Project
99
#
1010
# Redistribution and use in source and binary forms, with or without
1111
# modification, are permitted provided that the following conditions
@@ -65,6 +65,42 @@ if {[catch {mportinit "" "" ""} result]} {
6565
error "Failed to initialize ports system: $result"
6666
}
6767

68+
set archive_site_private ""
69+
set archive_site_public ""
70+
set jobs_dir ""
71+
set license_db_dir ""
72+
while {[string range [lindex $::argv 0] 0 1] eq "--"} {
73+
switch -- [lindex $::argv 0] {
74+
--archive_site_private {
75+
set archive_site_private [lindex $::argv 1]
76+
set ::argv [lrange $::argv 1 end]
77+
}
78+
--archive_site_public {
79+
set archive_site_public [lindex $::argv 1]
80+
set ::argv [lrange $::argv 1 end]
81+
}
82+
--jobs_dir {
83+
set jobs_dir [lindex $::argv 1]
84+
set ::argv [lrange $::argv 1 end]
85+
}
86+
--license_db_dir {
87+
set license_db_dir [lindex $::argv 1]
88+
set ::argv [lrange $::argv 1 end]
89+
}
90+
default {
91+
error "unknown option: [lindex $::argv 0]"
92+
}
93+
}
94+
set ::argv [lrange $::argv 1 end]
95+
}
96+
97+
if {$jobs_dir ne ""} {
98+
source ${jobs_dir}/distributable_lib.tcl
99+
if {$license_db_dir ne ""} {
100+
init_license_db $license_db_dir
101+
}
102+
}
103+
68104
set is_64bit_capable [sysctl hw.cpu64bit_capable]
69105

70106
array set portdepinfo {}
@@ -114,15 +150,39 @@ while {$todo ne {}} {
114150
}
115151
}
116152

153+
set opened 0
117154
if {[info exists outputports($p)] && $outputports($p) == 1} {
118155
if {[info exists portinfo(replaced_by)]} {
119156
puts stderr "Excluding $portinfo(name) because it is replaced by $portinfo(replaced_by)"
120157
set outputports($p) 0
121158
} elseif {[info exists portinfo(known_fail)] && [string is true -strict $portinfo(known_fail)]} {
122159
puts stderr "Excluding $portinfo(name) because it is known to fail"
123160
set outputports($p) 0
124-
} elseif {$::macports::os_major <= 10} {
161+
} elseif {$archive_site_public ne ""} {
162+
# FIXME: support non-default variants
125163
if {![catch {mportopen $portinfo(porturl) [list subport $portinfo(name)] ""} result]} {
164+
set opened 1
165+
set workername [ditem_key $result workername]
166+
set archive_name [$workername eval get_portimage_name]
167+
if {![catch {curl getsize ${archive_site_public}/$portinfo(name)/${archive_name}} size] && $size > 0} {
168+
puts stderr "Excluding $portinfo(name) because it has already been built and uploaded to the public server"
169+
set outputports($p) 0
170+
}
171+
} else {
172+
puts stderr "Excluding $portinfo(name) because it failed to open: $result"
173+
set outputports($p) 0
174+
}
175+
if {$outputports($p) == 1 && $archive_site_private ne "" && $jobs_dir ne ""} {
176+
# FIXME: support non-default variants
177+
set results [check_licenses $portinfo(name) [list]]
178+
if {[lindex $results 0] == 1 && ![catch {curl getsize ${archive_site_private}/$portinfo(name)/${archive_name}} size] && $size > 0} {
179+
puts stderr "Excluding $portinfo(name) because it is not distributable and it has already been built and uploaded to the private server"
180+
set outputports($p) 0
181+
}
182+
}
183+
}
184+
if {$outputports($p) == 1 && $::macports::os_major <= 10} {
185+
if {$opened == 1 || ![catch {mportopen $portinfo(porturl) [list subport $portinfo(name)] ""} result]} {
126186
set supported_archs [_mportkey $result supported_archs]
127187
if {$::macports::os_arch eq "i386" && !${is_64bit_capable} && $supported_archs ne "" && ("x86_64" ni $supported_archs || "i386" ni $supported_archs)} {
128188
puts stderr "Excluding $portinfo(name) because the ${::macports::macosx_version}_x86_64 builder will build it"
@@ -159,6 +219,10 @@ while {$todo ne {}} {
159219
}
160220
}
161221

222+
if {$jobs_dir ne "" && $license_db_dir ne ""} {
223+
write_license_db $license_db_dir
224+
}
225+
162226
set portlist [list]
163227
foreach portname [lsort -dictionary [array names portdepinfo]] {
164228
if {[info exists portdepinfo($portname)]} {

0 commit comments

Comments
 (0)