1#!/usr/bin/env bash
2
3
4stderr() {
5 echo "$@" 1>&2
6}
7
8usage() {
9 b=$(basename "$0")
10 echo $b: ERROR: "$@" 1>&2
11
12 cat 1>&2 <<EOF
13
14DESCRIPTION
15
16 $(basename "$0") is the script to run continuous integration commands for
17 go-toml on unix.
18
19 Requires Go and Git to be available in the PATH. Expects to be ran from the
20 root of go-toml's Git repository.
21
22USAGE
23
24 $b COMMAND [OPTIONS...]
25
26COMMANDS
27
28benchmark [OPTIONS...] [BRANCH]
29
30 Run benchmarks.
31
32 ARGUMENTS
33
34 BRANCH Optional. Defines which Git branch to use when running
35 benchmarks.
36
37 OPTIONS
38
39 -d Compare benchmarks of HEAD with BRANCH using benchstats. In
40 this form the BRANCH argument is required.
41
42 -a Compare benchmarks of HEAD against go-toml v1 and
43 BurntSushi/toml.
44
45 -html When used with -a, emits the output as HTML, ready to be
46 embedded in the README.
47
48coverage [OPTIONS...] [BRANCH]
49
50 Generates code coverage.
51
52 ARGUMENTS
53
54 BRANCH Optional. Defines which Git branch to use when reporting
55 coverage. Defaults to HEAD.
56
57 OPTIONS
58
59 -d Compare coverage of HEAD with the one of BRANCH. In this form,
60 the BRANCH argument is required. Exit code is non-zero when
61 coverage percentage decreased.
62EOF
63 exit 1
64}
65
66cover() {
67 branch="${1}"
68 dir="$(mktemp -d)"
69
70 stderr "Executing coverage for ${branch} at ${dir}"
71
72 if [ "${branch}" = "HEAD" ]; then
73 cp -r . "${dir}/"
74 else
75 git worktree add "$dir" "$branch"
76 fi
77
78 pushd "$dir"
79 go test -covermode=atomic -coverpkg=./... -coverprofile=coverage.out.tmp ./...
80 grep -Ev '(fuzz|testsuite|tomltestgen|gotoml-test-decoder|gotoml-test-encoder)' coverage.out.tmp > coverage.out
81 go tool cover -func=coverage.out
82 echo "Coverage profile for ${branch}: ${dir}/coverage.out" >&2
83 popd
84
85 if [ "${branch}" != "HEAD" ]; then
86 git worktree remove --force "$dir"
87 fi
88}
89
90coverage() {
91 case "$1" in
92 -d)
93 shift
94 target="${1?Need to provide a target branch argument}"
95
96 output_dir="$(mktemp -d)"
97 target_out="${output_dir}/target.txt"
98 head_out="${output_dir}/head.txt"
99
100 cover "${target}" > "${target_out}"
101 cover "HEAD" > "${head_out}"
102
103 cat "${target_out}"
104 cat "${head_out}"
105
106 echo ""
107
108 target_pct="$(tail -n2 ${target_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%.*/\1/')"
109 head_pct="$(tail -n2 ${head_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%/\1/')"
110 echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%"
111
112 delta_pct=$(echo "$head_pct - $target_pct" | bc -l)
113 echo "Delta: ${delta_pct}"
114
115 if [[ $delta_pct = \-* ]]; then
116 echo "Regression!";
117
118 target_diff="${output_dir}/target.diff.txt"
119 head_diff="${output_dir}/head.diff.txt"
120 cat "${target_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${target_diff}"
121 cat "${head_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${head_diff}"
122
123 diff --side-by-side --suppress-common-lines "${target_diff}" "${head_diff}"
124 return 1
125 fi
126 return 0
127 ;;
128 esac
129
130 cover "${1-HEAD}"
131}
132
133bench() {
134 branch="${1}"
135 out="${2}"
136 replace="${3}"
137 dir="$(mktemp -d)"
138
139 stderr "Executing benchmark for ${branch} at ${dir}"
140
141 if [ "${branch}" = "HEAD" ]; then
142 cp -r . "${dir}/"
143 else
144 git worktree add "$dir" "$branch"
145 fi
146
147 pushd "$dir"
148
149 if [ "${replace}" != "" ]; then
150 find ./benchmark/ -iname '*.go' -exec sed -i -E "s|github.com/pelletier/go-toml/v2|${replace}|g" {} \;
151 go get "${replace}"
152 fi
153
154 export GOMAXPROCS=2
155 go test '-bench=^Benchmark(Un)?[mM]arshal' -count=10 -run=Nothing ./... | tee "${out}"
156 popd
157
158 if [ "${branch}" != "HEAD" ]; then
159 git worktree remove --force "$dir"
160 fi
161}
162
163fmktemp() {
164 if mktemp --version &> /dev/null; then
165 # GNU
166 mktemp --suffix=-$1
167 else
168 # BSD
169 mktemp -t $1
170 fi
171}
172
173benchstathtml() {
174python3 - $1 <<'EOF'
175import sys
176
177lines = []
178stop = False
179
180with open(sys.argv[1]) as f:
181 for line in f.readlines():
182 line = line.strip()
183 if line == "":
184 stop = True
185 if not stop:
186 lines.append(line.split(','))
187
188results = []
189for line in reversed(lines[2:]):
190 if len(line) < 8 or line[0] == "":
191 continue
192 v2 = float(line[1])
193 results.append([
194 line[0].replace("-32", ""),
195 "%.1fx" % (float(line[3])/v2), # v1
196 "%.1fx" % (float(line[7])/v2), # bs
197 ])
198# move geomean to the end
199results.append(results[0])
200del results[0]
201
202
203def printtable(data):
204 print("""
205<table>
206 <thead>
207 <tr><th>Benchmark</th><th>go-toml v1</th><th>BurntSushi/toml</th></tr>
208 </thead>
209 <tbody>""")
210
211 for r in data:
212 print(" <tr><td>{}</td><td>{}</td><td>{}</td></tr>".format(*r))
213
214 print(""" </tbody>
215</table>""")
216
217
218def match(x):
219 return "ReferenceFile" in x[0] or "HugoFrontMatter" in x[0]
220
221above = [x for x in results if match(x)]
222below = [x for x in results if not match(x)]
223
224printtable(above)
225print("<details><summary>See more</summary>")
226print("""<p>The table above has the results of the most common use-cases. The table below
227contains the results of all benchmarks, including unrealistic ones. It is
228provided for completeness.</p>""")
229printtable(below)
230print('<p>This table can be generated with <code>./ci.sh benchmark -a -html</code>.</p>')
231print("</details>")
232
233EOF
234}
235
236benchmark() {
237 case "$1" in
238 -d)
239 shift
240 target="${1?Need to provide a target branch argument}"
241
242 old=`fmktemp ${target}`
243 bench "${target}" "${old}"
244
245 new=`fmktemp HEAD`
246 bench HEAD "${new}"
247
248 benchstat "${old}" "${new}"
249 return 0
250 ;;
251 -a)
252 shift
253
254 v2stats=`fmktemp go-toml-v2`
255 bench HEAD "${v2stats}" "github.com/pelletier/go-toml/v2"
256 v1stats=`fmktemp go-toml-v1`
257 bench HEAD "${v1stats}" "github.com/pelletier/go-toml"
258 bsstats=`fmktemp bs-toml`
259 bench HEAD "${bsstats}" "github.com/BurntSushi/toml"
260
261 cp "${v2stats}" go-toml-v2.txt
262 cp "${v1stats}" go-toml-v1.txt
263 cp "${bsstats}" bs-toml.txt
264
265 if [ "$1" = "-html" ]; then
266 tmpcsv=`fmktemp csv`
267 benchstat -format csv go-toml-v2.txt go-toml-v1.txt bs-toml.txt > $tmpcsv
268 benchstathtml $tmpcsv
269 else
270 benchstat go-toml-v2.txt go-toml-v1.txt bs-toml.txt
271 fi
272
273 rm -f go-toml-v2.txt go-toml-v1.txt bs-toml.txt
274 return $?
275 esac
276
277 bench "${1-HEAD}" `mktemp`
278}
279
280case "$1" in
281 coverage) shift; coverage $@;;
282 benchmark) shift; benchmark $@;;
283 *) usage "bad argument $1";;
284esac
View as plain text