genlink: Rewrite genlink script from awk to python

This removes the dependency on awk and should fix libopencm3#732
python was already a dependency, for the irq generation

Author: noonien

README.md

@@ -40,9 +40,7 @@ _TIP_: Include this repository as a Git submodule in your project to make sure
 Prerequisites
 -------------
 
-Building requires Python (some code is generated).
-If your user application uses the (optional) dynamic linker script generator,
-you will (presently) need GNU AWK.  Please see https://github.com/libopencm3/libopencm3/issues/732
+Building requires Python (Some code is generated).
 
 **For Ubuntu/Fedora:**
 

ld/README

@@ -13,9 +13,8 @@ File contents
 
 * {ROOT}/ld/tests/* 		- Prepared tests for the testing of the script
 * {ROOT}/ld/devices.data	- Device database file
-* {ROOT}/ld/Makefile.linker	- Common makefile part
 * {ROOT}/ld/linker.ld.S		- Linker script template
-* {ROOT}/scripts/genlink.awk	- Device database file search script 
+* {ROOT}/scripts/genlink.py	- Device database file search script 
 * {ROOT}/scripts/genlinktest.sh	- Device database file search test script
 
 Principle of operation
@@ -29,26 +28,15 @@ is case insensitive.
 
 DEVICE=stm32f407vgt6
 
-Then, the user includes the file /ld/Makefile.linker exactly after the first
-target (usually all) has been defined.
-
-The script automatically modifies the $(LDSCRIPT) variable to meet the new
-generated script with <device part name>.ld in the project directory, and adds
-a rule to make it from the scratch.
-
-Making the script is done by looking to device database file for the needed
-definitions, and applying those definitions to the C preprocessor source
-linker.ld.S outputting the preprocessed ld file.
-
 Device database contains definitions of common sections and its origins for
 the linker preprocessor. Every definition is interpreted in the linker script
 template as a macro, and it can be used for conditional insertion of the device
 dependent stuff.
 
-The search in the device database is pattern-based, and using awk script
-genlink.awk. The awk script traverses the file as a tree, joining the options
-for the preprocessor together by single space. The awk script adds -D to each
-parameter for you.
+The search in the device database is pattern-based, and using python script
+genlink.py. The python script traverses the file as a tree, joining the options
+for the preprocessor together by single space. The python script adds -D to
+each parameter for you.
 
 Testing
 -------
@@ -69,7 +57,7 @@ The testing stops after all test cases are valid, or at first error found.
 Example of use
 --------------
 
-* See Makefile.example file
+* Check the documentation for the genlink module in /mk/README.
 
 Device database file structure
 ------------------------------
@@ -127,9 +115,8 @@ stm32 END
 --- queried chip name ---
 stm32f051c4t6
 
---- output of the awk script ---
--D_ROM=16K -D_RAM=4K -D_ROM_OFF=0x08000000 -D_RAM_OFF=0x20000000 \
--mcpu=cortex-m0 -mthumb
+--- output of the python script ---
+-D_ROM=16K -D_RAM=4K -D_ROM_OFF=0x08000000 -D_RAM_OFF=0x20000000
 
 The generated linker script file will contain sections rom and ram with 
 appropriate initialization code, specified in linker file source linker.ld.S
@@ -154,4 +141,4 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public License
-along with this library. If not, see <http://www.gnu.org/licenses/>.
+along with this library. If not, see <http://www.gnu.org/licenses/>.

ld/devices.data

@@ -55,7 +55,7 @@
 # --- queried chip name ---
 # stm32f051c8t6
 #
-# --- output of the awk script ---
+# --- output of the python script ---
 # -DROM=16K -DRAM=4K -DROM_OFF=0x08000000 -DRAM_OFF=0x20000000
 #
 # The generated linker script file will contain sections rom and ram with

mk/genlink-config.mk

@@ -24,11 +24,11 @@ endif
 LDSCRIPT	= generated.$(DEVICE).ld
 DEVICES_DATA = $(OPENCM3_DIR)/ld/devices.data
 
-genlink_family		:=$(shell gawk -v PAT="$(DEVICE)" -v MODE="FAMILY" -f $(OPENCM3_DIR)/scripts/genlink.awk $(DEVICES_DATA))
-genlink_subfamily	:=$(shell gawk -v PAT="$(DEVICE)" -v MODE="SUBFAMILY" -f $(OPENCM3_DIR)/scripts/genlink.awk $(DEVICES_DATA))
-genlink_cpu		:=$(shell gawk -v PAT="$(DEVICE)" -v MODE="CPU" -f $(OPENCM3_DIR)/scripts/genlink.awk $(DEVICES_DATA))
-genlink_fpu		:=$(shell gawk -v PAT="$(DEVICE)" -v MODE="FPU" -f $(OPENCM3_DIR)/scripts/genlink.awk $(DEVICES_DATA))
-genlink_cppflags	:=$(shell gawk -v PAT="$(DEVICE)" -v MODE="CPPFLAGS" -f $(OPENCM3_DIR)/scripts/genlink.awk $(DEVICES_DATA))
+genlink_family		:=$(shell $(OPENCM3_DIR)/scripts/genlink.py $(DEVICES_DATA) $(DEVICE) FAMILY)
+genlink_subfamily	:=$(shell $(OPENCM3_DIR)/scripts/genlink.py $(DEVICES_DATA) $(DEVICE) SUBFAMILY)
+genlink_cpu		:=$(shell $(OPENCM3_DIR)/scripts/genlink.py $(DEVICES_DATA) $(DEVICE) CPU)
+genlink_fpu		:=$(shell $(OPENCM3_DIR)/scripts/genlink.py $(DEVICES_DATA) $(DEVICE) FPU)
+genlink_cppflags	:=$(shell $(OPENCM3_DIR)/scripts/genlink.py $(DEVICES_DATA) $(DEVICE) CPPFLAGS)
 
 CPPFLAGS	+= $(genlink_cppflags)
 

mk/genlink-rules.mk

@@ -19,4 +19,4 @@
 
 $(LDSCRIPT): $(OPENCM3_DIR)/ld/linker.ld.S $(OPENCM3_DIR)/ld/devices.data
 	@printf "  GENLNK  $(DEVICE)\n"
-	$(Q)$(CPP) $(ARCH_FLAGS) $(shell gawk -v PAT="$(DEVICE)" -v MODE="DEFS" -f $(OPENCM3_DIR)/scripts/genlink.awk $(OPENCM3_DIR)/ld/devices.data 2>/dev/null) -P -E $< > $@
+	$(Q)$(CPP) $(ARCH_FLAGS) $(shell $(OPENCM3_DIR)/scripts/genlink.py $(DEVICES_DATA) $(DEVICE) DEFS) -P -E $< > $@

scripts/genlink.awk

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# This python program generates parameters for the linker script generator feature.
+
+# This file is part of the libopencm3 project.
+#
+# 2017 George-Cristian Jiglau <noonien>
+#
+# This library is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library. If not, see <http://www.gnu.org/licenses/>.
+from __future__ import print_function
+import fnmatch
+import sys
+import re
+import os
+
+if len(sys.argv) != 4:
+    print("usage: %s <device-data-file> <device> <mode>" % sys.argv[0], file=sys.stderr)
+    sys.exit(1)
+
+data_file_path = sys.argv[1]
+find = sys.argv[2].lower()
+mode = sys.argv[3].upper()
+
+device = {
+    'info': {},
+    'defs': [],
+    'family': [],
+}
+
+# open device data file
+with open(data_file_path, 'r') as data_file:
+    # iterate lines
+    for line in data_file:
+        # strip whitespace from the beginning and end of line
+        line = line.strip()
+
+        # skip empty lines and comments
+        if line == '' or line.startswith('#'):
+            continue
+
+        # split line into it's parts: <pattern> <parent> <data..>
+        parts = line.split()
+        pattern, parent, data = parts[0], parts[1], parts[2:]
+
+        # skip line if pattern did not match first element
+        if not fnmatch.fnmatch(find, pattern):
+            continue
+
+        # extract data
+        for d in data:
+            # split into K=V
+            try:
+                (k, v) = d.split('=')
+            except:
+                continue
+
+            # skip invalid datas
+            if not re.match('^[A-Z0-9_]+$', k):
+                continue
+
+            # add FPU and CPU to info, not defs
+            if k in ('FPU', 'CPU'):
+                device['info'][k.lower()] = v
+                continue
+
+            device['defs'].append((k, v))
+
+        # if parent is +, there's more data for this pattern
+        if parent == '+':
+            continue
+
+        # device family
+        device['family'].append(find)
+
+        # break if this was the last line in this chain
+        if parent == 'END':
+            break
+
+        # look for the parent
+        find = parent
+
+# reverse device list
+device['family'] = device['family'][::-1]
+
+# device was not found
+if len(device['family']) == 0:
+    sys.exit(1)
+
+# for CPPFLAGS and DEFS, define device family
+if mode in ('CPPFLAGS', 'DEFS'):
+    sys.stdout.write(' '.join('-D%s' % d.upper() for d in device['family']))
+
+# defines
+if mode == 'DEFS':
+    if len(device['defs']) > 0:
+        defs = ' '.join('-D_%s=%s' % d for d in device['defs'])
+        sys.stdout.write(' ' + defs)
+
+# device family
+elif mode == 'FAMILY':
+    if len(device['family']) > 0:
+        sys.stdout.write(device['family'][0])
+
+# device subfamily
+elif mode == 'SUBFAMILY':
+    if len(device['family']) > 1:
+        sys.stdout.write(device['family'][1])
+
+# device info
+else:
+    info = mode.lower()
+    if info in device['info']:
+        sys.stdout.write(device['info'][info])
+
+sys.stdout.flush()

scripts/genlink.py

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-# This script is intended to test the awk program genlink.awk for the linker 
+# This script is intended to test the python program genlink.py for the linker 
 # script generator feature.
 #
 # See ld/README file for more info.
@@ -25,8 +25,13 @@
 # along with this library. If not, see <http://www.gnu.org/licenses/>.
 
 # run test
-PAAT=`basename $1`;
-gawk -v PAT="$PAAT" -f scripts/genlink.awk $1.data > $1.out;
+DEVICE=`basename $1`;
+(scripts/genlink.py $1.data $DEVICE CPPFLAGS; echo) > $1.out
+(scripts/genlink.py $1.data $DEVICE DEFS; echo) >> $1.out
+(scripts/genlink.py $1.data $DEVICE FAMILY; echo) >> $1.out
+(scripts/genlink.py $1.data $DEVICE SUBFAMILY; echo) >> $1.out
+(scripts/genlink.py $1.data $DEVICE CPU; echo) >> $1.out
+(scripts/genlink.py $1.data $DEVICE FPU; echo) >> $1.out
 
 #check test
 if ! diff -q $1.out $1.result >/dev/null; then
@@ -36,4 +41,4 @@ fi
 #remove workout only if it is OK
 rm -f $1.out
 
-exit 0  
+exit 0