path: root/compute_scripts.awk

#!/usr/bin/awk -f
#
# SPDX-License-Identifier: CC0-1.0
#
# Process javascript files and resolve dependencies between them
#
# This file is part of Haketilo
#
# Copyright (C) 2021, Wojtek Kosior
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the CC0 1.0 Universal License as published by
# the Creative Commons Corporation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# CC0 1.0 Universal License for more details.

BEGIN {
    true = 1
    false = 0
}

BEGIN {
    identifier_re    = "[_a-zA-Z][_a-zA-Z0-9]*"
    path_dir_re      = "([-_a-zA-Z0-9][-._a-zA-Z0-9]*/)*"
    path_ext_re      = "(\\.[-_.a-zA-Z0-9]*)?"
    path_re_noanchor = path_dir_re identifier_re path_ext_re
    path_re          = "^" path_re_noanchor "$"

    if_clause_re = "!?" identifier_re
    if_AND_re    = "([[:space:]]+&&[[:space:]]+" if_clause_re ")*"
    if_OR_re     = "([[:space:]]+[|][|][[:space:]]+" if_clause_re ")*"

    directive_args_patterns["IF"]      = ("^" if_clause_re \
					  "(" if_AND_re "|" if_OR_re ")$")
    directive_args_patterns["ENDIF"]   = "^$"
    directive_args_patterns["ELSE"]    = "^$"
    directive_args_patterns["ELIF"]    = "^(NOT[[:space:]]+)?" identifier_re "$"
    directive_args_patterns["DEFINE"]  = "^" identifier_re "$"
    directive_args_patterns["UNDEF"]   = "^" identifier_re "$"
    directive_args_patterns["ERROR"]   = "^.*$"
    directive_args_patterns["COPY"]    = "^[^[:space:]]+$"
    directive_args_patterns["INCLUDE"] = "^[^[:space:]]+$"
    directive_args_patterns["INCLUDE_VERBATIM"] = "^[^[:space:]]+$"

    AS_re                = "AS[[:space:]]+" identifier_re
    maybe_AS_re          = "([[:space:]]+" AS_re ")?"
    FROM_clause_re       = identifier_re maybe_AS_re
    more_FROM_clauses_re = "([[:space:]]*,[[:space:]]*" FROM_clause_re ")*"
    FROM_IMPORT_re       = "[^[:space:]]+[[:space:]]+IMPORT[[:space:]]+"
    EXPORT_AS_re         = ".*[^[:space:]][[:space:]]+" AS_re

    directive_args_patterns["IMPORT"] = "^[^[:space:]]+" maybe_AS_re "$"
    directive_args_patterns["FROM"]   = ("^" FROM_IMPORT_re FROM_clause_re \
					 more_FROM_clauses_re "$")
    directive_args_patterns["EXPORT"] = "^(" EXPORT_AS_re "|" identifier_re ")$"

    directive_args_patterns["LOADJS"] = "^[^[:space:]]+$"
    directive_args_patterns["LOADCSS"] = "^[^[:space:]]+$"

    directive_args_patterns["LOADHTML"] = "^[^[:space:]]+$"
}

function validate_path(read_path, path, line) {
    if (path !~ (path_re)) {
	printf "ERROR: File path in %s does not match '%s': %s\n",
	    read_path, path_re, line > "/dev/stderr"
	return 1
    }

    return 0
}

function identifier_from_path(path) {
    sub("^" path_dir_re, "", path)
    sub(path_ext_re "$", "", path)

    return path
}

function last_token(line) {
    sub("^.*[[:space:]]", "", line)

    return line
}

function first_token(line) {
    sub("[[:space:]].*$", "", line)

    return line
}

function is_empty(array,    key) {
    for (key in array)
	return false
    return true
}

function clear_array(array,    key) {
    for (key in array)
	delete array[key]
}

function add_line(path, line, where) {
    if (where != "amalgamation_root_file")
	lines[path,++lines_count[path]] = line

    if (where != "non_amalgamation_file" &&
	path == js_to_amalgamate)
	main_js_lines[++main_js_lines_count] = line
}

BEGIN {
    delete page_js_deps[0]
    page_js_deps_count = 0
}

function process_file(path, read_path, mode,
		      line, result, line_part, directive, directive_args,
		      if_nesting, if_nesting_true, if_branch_processed) {
    if (path in modes && modes[path] != mode) {
	printf "ERROR: File %s used multiple times in different contexts\n",
	    path > "/dev/stderr"
	return 1
    }

    if (mode == "html" && path == read_path) {
	clear_array(page_js_deps)
	page_js_deps_count = 0
	clear_array(page_css_deps)
    }

    modes[path] = mode

    if (!(path in reading)) {
	if (path in lines_count)
	    return 0
	lines_count[path]
    }

    reading[read_path]

    if (mode == "js" && path == read_path) {
	add_line(path, "\"use strict\";")
	add_line(path, "this.haketilo_exports = this.haketilo_exports || {};")
	add_line(path, "this.haketilo_exports[\"" path "\"] = {};")

	add_line(path, "window.globalThis = this", "amalgamation_root_file")

	add_line(path, "")

	add_line(path, "(function() {", "non_amalgamation_file")
	add_line(path, "var globalThis = this.haketilo_this",
		                        "non_amalgamation_file")
	add_line(path, "{",             "non_amalgamation_file")
    }

    while (true) {
	result = (getline line < read_path)
	if (result < 0) {
	    printf "ERROR: Could not read %s\n", read_path > "/dev/stderr"
	    return 1
	}
	if (result == 0) {
	    if (!(path in appended_lines_counts) || \
		additional_line_nr[path] == appended_lines_counts[path])
		break

	    line = appended_lines[path,++additional_line_nr[path]]
	}

	if (line !~ /^#/) {
	    if (if_nesting_true == if_nesting)
		add_line(path, line)
	    continue
	}

	while (line ~ /\\$/) {
	    sub(/\\$/, "", line)

	    result = (getline line_part < read_path)
	    if (result < 0) {
		printf "ERROR: Could not read %s\n", read_path > "/dev/stderr"
		return 1
	    }
	    if (result == 0) {
		if (path in appended_lines_counts && \
		    additional_line_nr[path] < appended_lines_counts[path]) {
		    line_part = appended_lines[path,++additional_line_nr[path]]
		} else {
		    printf "ERROR: Unexpected EOF in %s\n",
			read_path > "/dev/stderr"
		    return 1
		}
	    }

	    line = line " " line_part
	}

	directive = substr(line, 2)
	sub(/[[:space:]].*$/, "", directive)

	if (directive !~ \
	    /^(IF|ENDIF|ELSE|ELIF|DEFINE|UNDEF|ERROR|INCLUDE|INCLUDE_VERBATIM|COPY_FILE)$/ &&
	    (mode != "js" || directive !~ /^(IMPORT|FROM|EXPORT)$/) &&
	    (mode != "html" || directive !~ /^(LOADJS|LOADCSS)$/) &&
	    (mode != "manifest" || directive !~ /^(LOADJS|LOADHTML)$/)) {
	    printf "ERROR: Invalid # directive in %s: %s\n",
		read_path, line > "/dev/stderr"
	    return 1
	}

	directive_args = line
	sub(/^#[^[:space:]]*[[:space:]]*/, "", directive_args)
	sub(/[[:space:]]*$/, "", directive_args)

	if (directive_args !~ directive_args_patterns[directive]) {
	    printf "ERROR: #%s arguments in %s do not match '%s': %s\n",
		directive, read_path, directive_args_patterns[directive], line \
		> "/dev/stderr"
	    return 1
	}

	if (directive == "IF") {
	    if (if_nesting_true == if_nesting) {
		if (if_condition_true(directive_args, path))
		    if_nesting_true++
		else
		    if_branch_processed = false
	    }

	    if_nesting++
	} else if (directive == "ENDIF") {
	    if (if_nesting == 0) {
		printf "ERROR: Spurious #ENDIF in %s\n",
		    read_path > "/dev/stderr"
		return 1
	    }

	    if (if_nesting_true == if_nesting)
		if_nesting_true--

	    if_nesting--
	} else if (directive == "ELSE") {
	    if (if_nesting == 0) {
		printf "ERROR: Spurious #ELSE in %s\n",
		    read_path > "/dev/stderr"
		return 1
	    }

	    if (if_nesting == if_nesting_true + 1 && !if_branch_processed) {
		if_nesting_true++
	    } else if (if_nesting == if_nesting_true) {
		if_branch_processed = true
		if_nesting_true--
	    }
	} else if (directive == "ELIF") {
	    if (if_nesting == 0) {
		printf "ERROR: Spurious #ELIF in %s\n",
		    read_path > "/dev/stderr"
		return 1
	    }

	    if (if_nesting == if_nesting_true + 1 && !if_branch_processed &&
		if_condition_true(directive_args, path)) {
		if_nesting_true++
	    } else if (if_nesting == if_nesting_true) {
		if_branch_processed = true
		if_nesting_true--
	    }
	} else if (if_nesting_true != if_nesting) {
	    continue
	} else if (directive == "DEFINE") {
	    defines[path,directive_args]
	} else if (directive == "UNDEF") {
	    delete defines[path,directive_args]
	} else if (directive == "ERROR") {
	    printf "ERROR: File %s says: %s\n",
		read_path, directive_args > "/dev/stderr"
	    return 1
	} else if (directive == "INCLUDE") {
	    if (include_file(path, read_path, directive_args, line))
		return 1
	} else if (directive == "INCLUDE_VERBATIM") {
	    if (include_file(path, read_path, directive_args, line, true))
		return 1
	} else if (directive == "COPY_FILE") {
	    if (mark_copy_file(path, read_path, directive_args, line))
		return 1
	} else if (directive == "IMPORT") {
	    if (import_js_file(path, read_path, directive_args, line))
		return 1
	} else if (directive == "FROM") {
	    if (import_from_js_file(path, read_path, directive_args, line))
		return 1
	} else if (directive == "EXPORT") {
	    if (export_from_js_file(path, read_path, directive_args, line))
		return 1
	} else if (directive == "LOADJS") {
	    if (mode == "html") {
		page_js_deps_count = \
		    load_js_file(path, read_path, directive_args, line,
				 page_js_deps, page_js_deps_count)
		if (page_js_deps_count < 1)
		    return 1
	    } else if (mode == "manifest") {
		if (load_js_file(path, read_path, directive_args, line) < 1)
		    return 1
	    }
	} else if (directive == "LOADCSS") {
	    if (load_css_file(path, read_path, directive_args, line,
			      page_css_deps))
		return 1
	} else if (directive == "LOADHTML") {
	    if (load_html_file(path, read_path, directive_args, line))
		return 1
	}
    }

    close(read_path)

    if (if_nesting) {
	printf "ERROR: Unterminated #IF in %s\n", read_path > "/dev/stderr"
	return 1
    }

    if (mode == "js" && path == read_path) {
	add_line(path, "}",         "non_amalgamation_file")
	add_line(path, "}).call({", "non_amalgamation_file")
	add_line(path, "    haketilo_exports: this.haketilo_exports,",
		                    "non_amalgamation_file")
	add_line(path, "    haketilo_this:    this",
		                    "non_amalgamation_file")
	add_line(path, "});",       "non_amalgamation_file")
    }

    delete reading[read_path]
}

function if_condition_true(directive_args, path,
			   result, bool, first_iter, word, negated, alt) {
    first_iter = true

    while (directive_args) {
	word = first_token(directive_args)
	sub(/^[^[:space:]]+[[:space:]]*/, "", directive_args)
	alt = alt || directive_args ~ /^[|][|]/
	sub(/^[^[:space:]]+[[:space:]]*/, "", directive_args)

	negated = word ~ /^!/
	sub(/^!/, "", word)
	bool = (word in defines || (path,word) in defines) != negated

	if (first_iter) {
	    result = bool
	    first_iter = false
	    continue
	}

	if (alt)
	    result = result || bool
	else # if (directive_args ~ /^AND/)
	    result = result && bool
    }

    return result
}

function include_file(root_path, read_path, included_path, line, verbatim,
		      read_line, result) {
    if (included_path in reading) {
	printf "ERROR: Inclusion loop when including %s in %s\n",
	    included_path, read_path > "/dev/stderr"
	return 1
    }

    if (verbatim) {
	while(true) {
	    result = (getline read_line < included_path)
	    if (result > 0)
		add_line(root_path, read_line)
	    else
		break
	}

	if (result == 0) {
	    close(included_path)
	    return 0
	}

	printf "ERROR: Could not read %s\n", included_path > "/dev/stderr"
    } else {
	if (process_file(root_path, included_path, modes[root_path]) == 0)
	    return 0
    }

    printf "       when including %s in %s\n",
	included_path, read_path > "/dev/stderr"

    return 1
}

function mark_copy_file(root_path, read_path, copied_path, line) {
    if (validate_path(read_path, copied_path, line))
	return 1

    to_copy[copied_path]

    return 0
}

function satisfy_import(root_path, imported_path, as, what,
			added_line, description, count) {
    if ((root_path,as) in imports_from) {
	printf "ERROR: Multiple items imported under the name '%s' in %s\n",
	    as, root_path > "/dev/stderr"
	return 1
    }

    added_line = " " as " = haketilo_exports[\"" imported_path "\"]"
    if (what)
	added_line = added_line "." what

    add_line(root_path, "const" added_line ";", "non_amalgamation_file")
    add_line(root_path, "let"   added_line ";", "amalgamation_root_file")

    count = ++import_counts[root_path]

    imports_as  [root_path,count] = as
    imports_from[root_path,as]    = imported_path
    imports_what[root_path,as]    = what

    if (what)
	description = "'" what "' from " imported_path
    else
	description = imported_path

    description = description " needed by " root_path

    if (imported_path in reading) {
	printf "ERROR: dependency loop when importing %s\n",
	    description > "/dev/stderr"
	return 1
    } else if (process_file(imported_path, imported_path, "js")) {
	printf "       when importing %s\n", description > "/dev/stderr"
	return 1
    }

    if (what && !((imported_path,what) in exports)) {
	printf "ERROR: %s doesn't export '%s' needed by %s\n",
	    imported_path, what, root_path > "/dev/stderr"
	return 1
    }

    return 0
}

function import_js_file(root_path, read_path, directive_args, line,
			imported_path, as) {
    imported_path = first_token(directive_args)
    if (validate_path(read_path, imported_path, line))
	return 1

    if (line ~ (AS_re "$"))
	as = last_token(directive_args)
    else
	as = identifier_from_path(imported_path)

    return satisfy_import(root_path, imported_path, as)
}

function import_from_js_file(root_path, read_path, directive_args, line,
			     imported_path, args_copy, FROM_clause, as) {
    imported_path = first_token(directive_args)
    if (validate_path(read_path, imported_path, line))
	return 1

    args_copy = directive_args
    sub("^" FROM_IMPORT_re, "", args_copy)
    args_copy = "," args_copy

    while (args_copy ~ /,/) {
	sub(/^[^,]*,[[:space:]]*/, "", args_copy)

	FROM_clause = args_copy
	sub(/[[:space:]]*,.*$/, "", FROM_clause)

	if (satisfy_import(root_path, imported_path,
			   last_token(FROM_clause), first_token(FROM_clause)))
	    return 1
    }

    return 0
}

function export_from_js_file(root_path, read_path, directive_args, line,
			     as, exported_item, added_line) {
    as = last_token(directive_args)

    if (directive_args ~ ("^" identifier_re "$")) {
	exported_item = as
    } else {
	exported_item = directive_args
	sub("[[:space:]]+" AS_re "$", "", exported_item)
    }

    if ((root_path,as) in exports) {
	printf "ERROR: Multiple values exported under the name '%s' in %s\n",
	    as, root_path > "/dev/stderr"
	return 1
    }

    added_line = \
    "this.haketilo_exports[\"" root_path "\"]." as " = (" exported_item ");"
    add_line(root_path, added_line)

    exports[root_path,as]

    return 0
}

function compute_deps(js_path, dependencies, count, dependencies_added,
		      i_max, i, as, next_path) {
    delete dependencies_added[0]

    if (process_file(js_path, js_path, "js"))
	return 0

    i_max = import_counts[js_path]
    for (i = 1; i <= i_max; i++) {
	as = imports_as[js_path,i]
	next_path = imports_from[js_path,as]
	if (next_path in dependencies_added)
	    continue

	count = compute_deps(next_path, dependencies, count, dependencies_added)
	if (count < 1)
	    return 0
    }

    dependencies_added[js_path]
    dependencies[++count] = js_path

    return count
}

# Here js_deps and js_deps_count are optional args, used when loading scripts
# into an HTML page to avoid having the same script loaded twice in multiple
# places.
function load_js_file(root_path, read_path, loaded_path, line,
		      js_deps, js_deps_count,
		      js_deps_already_added, i, added_line) {
    delete js_deps[""]
    delete js_deps_already_added[0]

    if (validate_path(read_path, loaded_path, line))
	return 0

    for (i = 1; i <= js_deps_count; i++)
	js_deps_already_added[js_deps[i]]

    i = js_deps_count

    js_deps_count = compute_deps(loaded_path, js_deps,
				 js_deps_count, js_deps_already_added)

    if (js_deps_count < 1) {
	printf "       when loading %s from %s\n",
	    loaded_path, read_path > "/dev/stderr"
	return 0
    }

    while (++i <= js_deps_count) {
	if (modes[root_path] == "html") {
	    added_line = "<script src=\"/" js_deps[i] "\"></script>"
	} else { #if (modes[root_path] == "manifest") {
	    added_line = "\"" js_deps[i] "\""
	    if (i != js_deps_count)
		added_line = added_line ","
	}
	add_line(root_path, added_line)
    }

    return js_deps_count
}

# css_deps is an array used to avoid having the same stylesheet loaded twice in
# multiple places in a single HTML page.
function load_css_file(root_path, read_path, loaded_path, line, css_deps) {
    delete css_deps[""]

    if (validate_path(read_path, loaded_path, line))
	return 1

    if (!(loaded_path in css_deps)) {
	css_deps[loaded_path]
	to_copy[loaded_path]
	added_line = ("<link rel=\"stylesheet\" type=\"text/css\" "	\
		      "href=\"/" loaded_path "\" />")
	add_line(root_path, added_line)
    }

    return 0
}

function load_html_file(root_path, read_path, loaded_path, line) {
    if (validate_path(read_path, loaded_path, line))
	return 1

    if (process_file(loaded_path, loaded_path, "html")) {
	printf "       when loading %s from %s\n",
	    loaded_path, read_path, line > "/dev/stderr"
	return 1
    }

    return 0
}

function print_amalgamation(js_deps, js_deps_count,
			    js_dep_nr, path, max_line_nr, line_nr) {
    delete js_deps[0]

    js_deps_count = compute_deps(js_to_amalgamate, js_deps, 0)
    if (js_deps_count < 1)
	return 1

    # '<' instead of '<=' because we print the main js file below instead
    for (js_dep_nr = 1; js_dep_nr < js_deps_count; js_dep_nr++) {
	path = js_deps[js_dep_nr]
	max_line_nr = lines_count[path]

	for (line_nr = 1; line_nr <= max_line_nr; line_nr++)
	    print lines[path, line_nr]
    }

    for (line_nr = 1; line_nr <= main_js_lines_count; line_nr++)
	    print main_js_lines[line_nr]

    return 0
}

function print_usage() {
    printf "USAGE: %s compute_scripts.awk -- [-D PREPROCESSOR_DEFINITION]... [-M manifest/to/process/manifest.json]... [-H html/to/process.html]... [-J js/to/process.js]... [-A file/to/append/to.js:appended_code]... [--help|-h] [--output-dir=./build] [--write-js-deps] [--write-html-deps] [--output=files-to-copy|--output=amalgamate-js:js/to/process.js]\n",
	ARGV[0] > "/dev/stderr"
}

BEGIN {
    option_arg_patterns["D"] = "^" identifier_re "$"
    option_arg_patterns["M"] = path_re
    option_arg_patterns["H"] = path_re
    option_arg_patterns["J"] = path_re
    option_arg_patterns["A"] = "^" path_re_noanchor ":"
}

function main(i, j, path, letter, dir, max_line_nr, js_deps, js_deps_count,
	      code, tmp_lines) {
    output_dir = "./build"
    write_js_deps = false
    write_html_deps = false

    delete appended_lines[0]
    delete appended_lines_counts[0]
    delete tmp_lines[0]

    delete lines[0]
    delete lines_count[0]

    output = ""
    js_to_amalgamate = ""
    delete main_js_lines[0]

    delete manifests_to_process[0]
    delete html_to_process[0]
    delete js_to_process[0]

    delete explicitly_requested[0]

    for (i = 1; i < ARGC; i++) {
	if (ARGV[i] ~ /^-[DMHJA]$/) {
	    letter = substr(ARGV[i++], 2)
	    if (i == ARGC || ARGV[i] !~ option_arg_patterns[letter]) {
		printf "ERROR: '-%s' option should be followed by an argument matching '%s'\n",
		    letter, option_arg_patterns[letter] > "/dev/stderr"
		return 1
	    }

	    if (letter == "D")
		defines[ARGV[i]]
	    else
		explicitly_requested[ARGV[i]]

	    if (letter == "M")
		manifests_to_process[ARGV[i]]
	    if (letter == "H")
		html_to_process[ARGV[i]]
	    if (letter == "J")
		js_to_process[ARGV[i]]

	    if (letter == "A") {
		path = ARGV[i]
		sub(/:.*$/, "", path)
		if (path in appended_lines_counts) {
		    printf "ERROR: The same file %s given to the '-A' option multiple times\n",
			path > "/dev/stderr"
		    return 1
		}

		clear_array(tmp_lines)
		code = ARGV[i]
		sub(/^[^:]+:/, "", code)
		appended_lines_counts[path] = split(code, tmp_lines, "\n")
		for (j = appended_lines_counts[path]; j > 0; j--)
		    appended_lines[path,j] = tmp_lines[j]
	    }
	} else if (ARGV[i] ~ /^-(-help|h)$/ ) {
	    print_usage()
	    return 0
	} else if (ARGV[i] ~ /^--output-dir=/) {
	    output_dir = ARGV[i]
	    sub(/^--output-dir=/, "", output_dir)
	} else if (ARGV[i] ~ /^--write-js-deps$/) {
	    write_js_deps = true
	} else if (ARGV[i] ~ /^--write-html-deps$/) {
	    write_html_deps = true
	} else if (ARGV[i] ~ /^--output=files-to-copy$/) {
	    output = "files-to-copy"
	} else if (ARGV[i] ~ /^--output=amalgamate-js:/) {
	    output = "amalgamate-js"
	    js_to_amalgamate = ARGV[i]
	    sub(/^--output=amalgamate-js:/, "", js_to_amalgamate)
	    if (js_to_amalgamate !~ path_re) {
		printf "ERROR: amalgamate-js path does not match '%s': %s\n",
		    path_re, js_to_amalgamate > "/dev/stderr"
		return 1
	    }
	} else {
		printf "ERROR: Unknown option '%s'\n", ARGV[i] > "/dev/stderr"
		print_usage()
		return 1
	}
    }

    if (is_empty(explicitly_requested) && output != "amalgamate-js") {
	explicitly_requested["manifest.json"]
	manifests_to_process["manifest.json"]
    }

    for (path in manifests_to_process) {
	if (process_file(path, path, "manifest"))
	    return 1
    }
    for (path in html_to_process) {
	if (process_file(path, path, "html"))
	    return 1
    }
    for (path in js_to_process) {
	if (process_file(path, path, "js"))
	    return 1
    }

    for (path in lines_count) {
	if (!(path in explicitly_requested) &&
	    !(modes[path] == "js" && write_js_deps) &&
	    !(modes[path] == "html" && write_html_deps))
	    continue

	dir = path
	sub(/[^/]*$/, "", dir)
	dir = output_dir "/" dir
	sub("'", "'\\''", dir)

	system("mkdir -p '" dir "'")

	printf "" > (output_dir "/" path)

	max_line_nr = lines_count[path]
	for (i = 1; i <= max_line_nr; i++)
	    print lines[path, i] >> (output_dir "/" path)
    }

    if (output == "files-to-copy") {
	for (path in to_copy)
	    print path
    }

    if (output == "amalgamate-js") {
	if (print_amalgamation())
	    return 1
    }

    return 0
}

BEGIN {
    exit main()
}