1 module gccbuild.config;
2 
3 import gccbuild, scriptlike, painlessjson, std.json;
4 
5 BuildMode mode = BuildMode.all;
6 Path buildConfig, sourceConfig, logFilePath;
7 string buildTriplet = "x86_64-unknown-linux-gnu";
8 Path mirrorFile = "mirrors.json";
9 bool verifyCachedSources = true;
10 bool forceDownload = false;
11 bool forceExtract = false;
12 size_t numCPUs = 1;
13 Path cacheDir;
14 string[string] buildVariables;
15 string[string] cmdVariables;
16 bool skipStripLibraries = false;
17 bool skipStripBinaries = false;
18 bool keepBuildFiles = false;
19 string hostStripCMD;
20 string[] patchDirsCMD;
21 string gdcSourcePath;
22 string targetGCCCMD;
23 
24 @property string targetGCC()
25 {
26     if (targetGCCCMD.empty)
27         return build.target ~ "-gcc";
28     else
29         return targetGCCCMD;
30 }
31 
32 @property string hostStrip()
33 {
34     if (!hostStripCMD.empty)
35         return hostStripCMD;
36     else if (build.type == ToolchainType.native)
37         return "strip";
38     else
39         return build.host ~ "-strip";
40 }
41 
42 Path[] patchDirectories;
43 
44 struct CMDBuildOverwrites
45 {
46     string host, target;
47     string gccFile, gccSuburl, gccMD5;
48     Nullable!ToolchainType type;
49 }
50 
51 CMDBuildOverwrites cmdOverwrites;
52 
53 @property Path sysrootDirStage1()
54 {
55     return toolchainDirStage1 ~ "sysroot";
56 }
57 
58 @property Path binDirStage1()
59 {
60     return toolchainDirStage1 ~ "bin";
61 }
62 
63 @property Path toolchainDirStage1()
64 {
65     return installDir ~ "stage1";
66 }
67 
68 @property Path sysrootDirWithPrefix()
69 {
70     return sysrootDir ~ build.relativeSysrootPrefix;
71 }
72 
73 @property Path sysrootDir()
74 {
75     return toolchainDir ~ "sysroot";
76 }
77 
78 @property Path binDir()
79 {
80     return toolchainDir ~ "bin";
81 }
82 
83 @property Path toolchainDir()
84 {
85     return installDir ~ build.target;
86 }
87 
88 @property Path hostlibDir()
89 {
90     return installDir ~ "host";
91 }
92 
93 @property Path mainPatchDir()
94 {
95     return cacheDir ~ "patches";
96 }
97 
98 @property Path installDir()
99 {
100     return cacheDir ~ "install";
101 }
102 
103 @property Path extractDir()
104 {
105     return cacheDir ~ "src";
106 }
107 
108 @property Path buildDir()
109 {
110     return cacheDir ~ "build";
111 }
112 
113 @property Path downloadDir()
114 {
115     return cacheDir ~ "download";
116 }
117 
118 MainConfig build;
119 string[][string] mirrors;
120 
121 void setDefaultPaths()
122 {
123     cacheDir = Path("cache").absolutePath;
124     logFilePath = Path("build.log").absolutePath;
125 }
126 
127 void loadMirrors()
128 {
129     startSectionLog("Loading mirror information");
130     try
131         mirrors = fromJSON!(typeof(mirrors))(mirrorFile.readText().parseJSON());
132     catch (Exception e)
133         failc("Couldn't load mirror file ", mirrorFile, ": ", e);
134     endSectionLog();
135 }
136 
137 void loadSourceConfig(Path file)
138 {
139     startSectionLog("Loading build sources configuration");
140     try
141         build = fromJSON!MainConfig(file.readText().parseJSON());
142     catch (Exception e)
143         failc("Couldn't load build sources configuration ", file, ": ", e);
144     endSectionLog();
145 }
146 
147 void loadBuildConfig(Path file)
148 {
149     startSectionLog("Loading build configuration");
150     BuildConfig commands;
151     try
152         commands = fromJSON!BuildConfig(file.readText().parseJSON());
153     catch (Exception e)
154         failc("Couldn't load build configuration ", file, ": ", e);
155 
156     build.include(commands);
157     build.include(cmdOverwrites);
158 
159     endSectionLog();
160 }
161 
162 void setupBuildVariables()
163 {
164     startSectionLog("Setting build variables");
165 
166     buildVariables["NUM_CPU"] = to!string(numCPUs);
167     buildVariables["BUILD"] = buildTriplet;
168     buildVariables["HOST"] = build.host;
169     buildVariables["TARGET"] = build.target;
170     buildVariables["ARCH"] = build.arch;
171     buildVariables["DIR_GMP_INSTALL"] = (hostlibDir ~ build.gmp.baseDirName).toString();
172     buildVariables["DIR_MPFR_INSTALL"] = (hostlibDir ~ build.mpfr.baseDirName).toString();
173     buildVariables["DIR_MPC_INSTALL"] = (hostlibDir ~ build.mpc.baseDirName).toString();
174     buildVariables["DIR_TOOLCHAIN"] = toolchainDir.toString();
175     buildVariables["DIR_SYSROOT"] = sysrootDir.toString();
176     buildVariables["DIR_SYSROOT_PREFIX"] = build.sysrootPrefix;
177     buildVariables["DIR_SYSROOT_WITH_PREFIX"] = sysrootDirWithPrefix.toString();
178     buildVariables["DIR_TOOLCHAIN_STAGE1"] = toolchainDirStage1.toString();
179     buildVariables["DIR_SYSROOT_STAGE1"] = sysrootDirStage1.toString();
180     buildVariables["DIR_SYSROOT_STAGE1_WITH_PREFIX"] = (
181         sysrootDirStage1 ~ build.relativeSysrootPrefix).toString();
182     buildVariables["TARGET_GCC"] = build.target ~ "-gcc";
183 
184     // overwrite from build.json
185     static void addConstants(string[string] cst)
186     {
187         foreach (key, val; cst)
188         {
189             buildVariables[key] = val;
190         }
191     }
192 
193     addConstants(build.constants);
194     final switch (build.type)
195     {
196     case ToolchainType.cross:
197         addConstants(build.constantsCross);
198         break;
199     case ToolchainType.native:
200         addConstants(build.constantsNative);
201         break;
202     case ToolchainType.cross_native:
203         addConstants(build.constantsCrossNative);
204         break;
205     case ToolchainType.canadian:
206         addConstants(build.constantsCanadian);
207         break;
208     }
209 
210     // overwrites from cmd
211     foreach (key, val; cmdVariables)
212     {
213         buildVariables[key] = val;
214     }
215 
216     foreach (key, val; buildVariables)
217     {
218         yap(key, "=", val);
219     }
220     endSectionLog();
221 
222     // Setup patch directories
223     startSectionLog("Setting patch directories");
224 
225     static void addConfPatchDirs(string[] dirs)
226     {
227         foreach (dir; dirs)
228         {
229             // Make sure relative Paths are relative to the build config specifying them
230             auto path = Path(dir);
231             if (!path.isAbsolute)
232                 path = path.absolutePath(buildConfig.dirName);
233             patchDirectories ~= path;
234         }
235     }
236 
237     patchDirectories ~= mainPatchDir;
238     addConfPatchDirs(build.localPatchDirs);
239     final switch (build.type)
240     {
241     case ToolchainType.cross:
242         addConfPatchDirs(build.localPatchDirsCross);
243         break;
244     case ToolchainType.native:
245         addConfPatchDirs(build.localPatchDirsNative);
246         break;
247     case ToolchainType.cross_native:
248         addConfPatchDirs(build.localPatchDirsCrossNative);
249         break;
250     case ToolchainType.canadian:
251         addConfPatchDirs(build.localPatchDirsCanadian);
252         break;
253     }
254     foreach (dir; patchDirsCMD)
255         patchDirectories ~= Path(dir);
256 
257     endSectionLog();
258 }
259 
260 void dumpConfiguration()
261 {
262     startSection("Dumping configuration");
263     writeBulletPoint(mixin(interp!"Type: ${build.type}     Target type: ${build.targetType}"));
264     writeBulletPoint(mixin(interp!"Build: ${buildTriplet}"));
265     writeBulletPoint(mixin(interp!"Host: ${build.host}"));
266     writeBulletPoint(mixin(interp!"Target: ${build.target}"));
267     if (build.targetType == HostType.linux)
268         writeBulletPoint(mixin(interp!"Kernel ARCH: ${build.arch}"));
269     endSection();
270 }
271 
272 auto namedFields(T, A...)(ref T instance)
273 {
274     static string generateMixin(string[] fields)
275     {
276         string result = "return only(";
277         foreach (i, entry; fields)
278         {
279             if (i != 0)
280                 result ~= ", ";
281             result ~= "tuple!(\"name\", \"value\")(\"" ~ entry ~ "\", cast(Component)instance." ~ entry ~ ")";
282         }
283         result ~= ");";
284         return result;
285     }
286 
287     mixin(generateMixin([A]));
288 }
289 
290 struct BuildConfig
291 {
292     string host, target, arch;
293     string[string] constants;
294     @SerializedName("constants_native") string[string] constantsNative;
295     @SerializedName("constants_cross") string[string] constantsCross;
296     @SerializedName("constants_cross_native") string[string] constantsCrossNative;
297     @SerializedName("constants_canadian") string[string] constantsCanadian;
298 
299     @SerializedName("sysroot_prefix") string sysrootPrefix = "/";
300     @SerializedName("patch_dirs") string[] localPatchDirs;
301     @SerializedName("patch_dirs_native") string[] localPatchDirsNative;
302     @SerializedName("patch_dirs_cross") string[] localPatchDirsCross;
303     @SerializedName("patch_dirs_cross_native") string[] localPatchDirsCrossNative;
304     @SerializedName("patch_dirs_canadian") string[] localPatchDirsCanadian;
305 
306     string type = "";
307 
308     BuildCommand gmp, mpfr, mpc, linux, binutils, w32api, gcc;
309     GlibcBuildCommand glibc;
310     CleanupCommand cleanup;
311     @SerializedName("cleanup_cross") CleanupCommand cleanupCross;
312     @SerializedName("cleanup_cross_native") CleanupCommand cleanupCrossNative;
313     @SerializedName("cleanup_native") CleanupCommand cleanupNative;
314     @SerializedName("cleanup_canadian") CleanupCommand cleanupCanadian;
315     @SerializedName("gcc_stage1") BuildCommand gccStage1;
316 }
317 
318 struct MainConfig
319 {
320     @SerializeIgnore string host;
321     @SerializeIgnore string target;
322     @SerializeIgnore string arch;
323     @SerializeIgnore string[string] constants;
324     @SerializeIgnore string[string] constantsNative;
325     @SerializeIgnore string[string] constantsCross;
326     @SerializeIgnore string[string] constantsCrossNative;
327     @SerializeIgnore string[string] constantsCanadian;
328 
329     @SerializeIgnore MultilibEntry[] multilibs;
330     @SerializeIgnore string sysrootPrefix;
331     @property string relativeSysrootPrefix()
332     {
333         return sysrootPrefix.relativePath("/");
334     }
335 
336     @SerializeIgnore ToolchainType type;
337     @SerializeIgnore string[] localPatchDirs;
338     @SerializeIgnore string[] localPatchDirsNative;
339     @SerializeIgnore string[] localPatchDirsCross;
340     @SerializeIgnore string[] localPatchDirsCrossNative;
341     @SerializeIgnore string[] localPatchDirsCanadian;
342 
343     Component mpc, mpfr, gmp, binutils, linux, w32api;
344     GCCComponent gcc;
345     GlibcComponent glibc;
346     // This is a special component: only a addon for glibc, don't build individually
347     @SerializedName("glibc_ports") Component glibcPorts;
348 
349     @SerializeIgnore CleanupCommand cleanup;
350     @SerializeIgnore CleanupCommand cleanupCross;
351     @SerializeIgnore CleanupCommand cleanupCrossNative;
352     @SerializeIgnore CleanupCommand cleanupNative;
353     @SerializeIgnore CleanupCommand cleanupCanadian;
354     @property CleanupCommand variantCleanup()
355     {
356         final switch (type)
357         {
358         case ToolchainType.native:
359             return cleanupNative;
360         case ToolchainType.canadian:
361             return cleanupCanadian;
362         case ToolchainType.cross:
363             return cleanupCross;
364         case ToolchainType.cross_native:
365             return cleanupCrossNative;
366         }
367     }
368 
369     @property HostType targetType()
370     {
371         if (target.toLower().canFind("mingw"))
372             return HostType.mingw;
373         else
374             return HostType.linux;
375     }
376 
377     @property componentRange()
378     {
379         return this.namedFields!(MainConfig, "mpc", "mpfr", "gmp", "glibc",
380             "binutils", "linux", "w32api", "gcc");
381     }
382 
383     @property configuredComponents()
384     {
385         return this.componentRange.filter!(a => a.value.isInConfig);
386     }
387 
388     void include(BuildConfig config)
389     {
390         host = config.host;
391         target = config.target;
392         arch = config.arch;
393         constants = config.constants;
394         constantsNative = config.constantsNative;
395         constantsCross = config.constantsCross;
396         constantsCrossNative = config.constantsCrossNative;
397         constantsCanadian = config.constantsCanadian;
398         sysrootPrefix = config.sysrootPrefix;
399         localPatchDirs = config.localPatchDirs;
400         localPatchDirsNative = config.localPatchDirsNative;
401         localPatchDirsCross = config.localPatchDirsCross;
402         localPatchDirsCrossNative = config.localPatchDirsCrossNative;
403         localPatchDirsCanadian = config.localPatchDirsCanadian;
404         cleanup = config.cleanup;
405         cleanupNative = config.cleanupNative;
406         cleanupCanadian = config.cleanupCanadian;
407         cleanupCross = config.cleanupCross;
408         cleanupCrossNative = config.cleanupCrossNative;
409 
410         void trySet(Component comp, BuildCommand com)
411         {
412             if (comp)
413                 comp._mainBuildCommand = com;
414         }
415 
416         trySet(mpc, config.mpc);
417         trySet(mpfr, config.mpfr);
418         trySet(gmp, config.gmp);
419         trySet(glibc, config.glibc);
420         trySet(binutils, config.binutils);
421         trySet(linux, config.linux);
422         trySet(w32api, config.w32api);
423         if (gcc)
424         {
425             gcc._mainBuildCommand = config.gcc;
426             gcc._stage1BuildCommand = config.gccStage1;
427         }
428     }
429 
430     void include(CMDBuildOverwrites overwrite)
431     {
432         void overwriteIfSet(ref string oldVal, string newVal)
433         {
434             if (!newVal.empty)
435                 oldVal = newVal;
436         }
437 
438         overwriteIfSet(host, cmdOverwrites.host);
439         overwriteIfSet(target, cmdOverwrites.target);
440         overwriteIfSet(gcc.file, cmdOverwrites.gccFile);
441         overwriteIfSet(gcc.suburl, cmdOverwrites.gccSuburl);
442         overwriteIfSet(gcc.md5, cmdOverwrites.gccMD5);
443 
444         if (build.target == build.host)
445         {
446             if (build.host == buildTriplet)
447                 build.type = ToolchainType.native;
448             else
449                 build.type = ToolchainType.cross_native;
450         }
451         else
452         {
453             if (build.host == buildTriplet)
454                 build.type = ToolchainType.cross;
455             else
456                 build.type = ToolchainType.canadian;
457         }
458 
459         if (!overwrite.type.isNull)
460             type = overwrite.type;
461     }
462 }
463 
464 class Component
465 {
466 private:
467     BuildCommand _mainBuildCommand;
468 
469 public:
470 
471     string file, url, md5;
472     // Total url is url | mirror ~ suburl | mirror ~ filename
473     string suburl;
474 
475     @property BuildCommand mainBuildCommand()
476     {
477         return _mainBuildCommand;
478     }
479 
480     @SerializeIgnore bool wasExtracted = false;
481 
482     @property Path localFile()
483     {
484         return downloadDir ~this.file;
485     }
486 
487     @property Path baseDirName()
488     {
489         return Path(file.stripExt.stripExt);
490     }
491 
492     Path getSourceFile(Path relFile)
493     {
494         return sourceFolder ~ relFile;
495     }
496 
497     @property Path configureFile()
498     {
499         return getSourceFile(Path("configure"));
500     }
501 
502     @property Path sourceFolder()
503     {
504         return extractDir ~ baseDirName;
505     }
506 
507     @property Path buildFolder()
508     {
509         return buildDir ~ baseDirName;
510     }
511 }
512 
513 // Whether this component is specified in config file
514 @property bool isInConfig(Component c)
515 {
516     return c !is null && c.mainBuildCommand !is null;
517 }
518 
519 @property bool hasBuildCommands(Component c)
520 {
521     return c.isInConfig && c.mainBuildCommand.matchesBuildType;
522 }
523 
524 class GlibcComponent : Component
525 {
526 public:
527     override @property GlibcBuildCommand mainBuildCommand()
528     {
529         return cast(GlibcBuildCommand) _mainBuildCommand;
530     }
531 }
532 
533 class GCCComponent : Component
534 {
535 private:
536     BuildCommand _stage1BuildCommand;
537 
538 public:
539     @property BuildCommand stage1BuildCommand()
540     {
541         return _stage1BuildCommand;
542     }
543 }
544 
545 enum ToolchainType
546 {
547     native,
548     cross,
549     cross_native,
550     canadian
551 }
552 
553 enum HostType
554 {
555     mingw,
556     linux
557 }
558 
559 enum BuildMode
560 {
561     all,
562     download,
563     extract,
564     patch
565 }
566 
567 class BuildCommand
568 {
569     string[] commands;
570     string[] variants;
571 }
572 
573 @property bool matchesBuildType(BuildCommand cmd)
574 {
575     if (!cmd)
576         return false;
577 
578     if (cmd.variants.empty)
579         return true;
580 
581     foreach (variant; cmd.variants)
582     {
583         if (variant == to!string(build.type))
584             return true;
585     }
586     return false;
587 }
588 
589 class GlibcBuildCommand : BuildCommand
590 {
591     @SerializedName("multi_commands") string[][] multiCommands;
592 }
593 
594 class CleanupCommand : BuildCommand
595 {
596     @SerializedName("strip_target") string[] stripTarget;
597     @SerializedName("strip_host") string[] stripHost;
598     string[] remove;
599 }
600 
601 struct MultilibEntry
602 {
603     string gccFolder;
604     string args;
605     string osFolder;
606 
607     @property bool isDefaultLib()
608     {
609         return gccFolder == ".";
610     }
611 }