Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

builtin.inc: fix build with -Woverlength-strings #2979

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ jobs:
--with-oniguruma=builtin \
--enable-static \
--enable-all-static \
HOSTCC=c99 \
CFLAGS="-O2 -pthread -fstack-protector-all"
make -j"$(nproc)"
file ./jq
Expand Down Expand Up @@ -159,6 +160,7 @@ jobs:
--with-oniguruma=builtin \
--enable-static \
--enable-all-static \
HOSTCC=c99 \
CFLAGS="-O2 -pthread -fstack-protector-all"
make -j"$(sysctl -n hw.logicalcpu)"
strip ./jq
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ jq
!tests/modules/lib/jq/
jq.1

# Generator programs
src/gen_jq_builtins

# Generated source
src/builtin.inc
src/config_opts.inc
Expand Down
27 changes: 18 additions & 9 deletions Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ LIBJQ_INCS = src/builtin.h src/bytecode.h src/compile.h \
src/linker.h src/locfile.h src/opcode_list.h src/parser.y \
src/util.h src/decNumber/decContext.h src/decNumber/decNumber.h \
src/decNumber/decNumberLocal.h src/jv_dtoa_tsd.h src/jv_thread.h \
src/jv_private.h
src/jv_private.h src/math-macos.h src/main-win32.h

LIBJQ_SRC = src/builtin.c src/bytecode.c src/compile.c src/execute.c \
src/jq_test.c src/jv.c src/jv_alloc.c src/jv_aux.c \
Expand All @@ -19,10 +19,13 @@ LIBJQ_SRC = src/builtin.c src/bytecode.c src/compile.c src/execute.c \

### C build options

AM_CFLAGS = -Wextra -Wall -Wno-unused-parameter -Wno-unused-function
AM_CFLAGS = -Wextra -Wall -Woverlength-strings

AM_CFLAGS += -Wno-unused-parameter -Wno-unused-function

if WIN32
AM_CFLAGS += -municode
HOSTCFLAGS += -municode
endif

ACLOCAL_AMFLAGS = -I config/m4
Expand Down Expand Up @@ -107,22 +110,27 @@ generate_ver = ver="`{ $(srcdir)/scripts/version || echo '$(VERSION)' ; } | sed
.remake-version-h: .FORCE
@ $(generate_ver); test "x`cat src/version.h 2>/dev/null`" = "x$$ver" || touch .remake-version-h
src/version.h: .remake-version-h
mkdir -p src
@mkdir -p src
$(AM_V_GEN) $(generate_ver); echo "$$ver" > $@
src/config_opts.inc:
mkdir -p src
@mkdir -p src
$(AM_V_GEN) if test -x ./config.status; then \
./config.status --config; \
else echo "(unknown)"; \
fi | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' -e 's/^/"/' -e 's/$$/"/' -e 's/^/#define JQ_CONFIG /' > $@
src/main.c: src/version.h src/config_opts.inc

src/builtin.inc: $(srcdir)/src/builtin.jq
mkdir -p src
$(AM_V_GEN) sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' -e 's/^/"/' -e 's/$$/\\n"/' $(srcdir)/src/builtin.jq > $@
src/gen_jq_builtins: $(srcdir)/src/gen_jq_builtins.c
@mkdir -p src
@printf ' HOSTCC %s\n' $@
@$(HOSTCC) $(HOSTCFLAGS) $(DEFS) -o $@ $<
src/builtin.inc: src/gen_jq_builtins $(srcdir)/src/builtin.jq
@mkdir -p src
$(AM_V_GEN)src/gen_jq_builtins $(srcdir)/src/builtin.jq > $@
src/builtin.o: src/builtin.inc

CLEANFILES = src/version.h .remake-version-h src/builtin.inc src/config_opts.inc
CLEANFILES = src/version.h .remake-version-h src/config_opts.inc \
src/gen_jq_builtins src/builtin.inc

bin_PROGRAMS = jq
jq_SOURCES = src/main.c src/version.h
Expand Down Expand Up @@ -232,7 +240,8 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER) \
tests/optional.test tests/man.test tests/manonig.test \
tests/jq.test tests/onig.test tests/base64.test \
tests/utf8-truncate.jq tests/jq-f-test.sh \
tests/no-main-program.jq tests/yes-main-program.jq
tests/no-main-program.jq tests/yes-main-program.jq \
src/gen_jq_builtins.c

AM_DISTCHECK_CONFIGURE_FLAGS=--with-oniguruma=builtin

Expand Down
6 changes: 3 additions & 3 deletions compile-ios.sh
Original file line number Diff line number Diff line change
Expand Up @@ -80,15 +80,15 @@ cd "${builddir}/"

# ./configure; make install
cd "${builddir}/onig-${oniguruma}"
CC=${CC} LDFLAGS=${LDFLAGS} \
HOSTCC=${HOSTCC:-c99} CC=${CC} LDFLAGS=${LDFLAGS} \
./configure --host=${HOST} --build=$(./config.guess) --enable-shared=no --enable-static=yes --prefix=/
make -j${MAKEJOBS} install DESTDIR="${cwd}/ios/onig/${arch}"
make clean

# Jump back to JQ.
cd ${cwd}
[[ ! -f ./configure ]] && autoreconf -ivf
CC=${CC} LDFLAGS=${LDFLAGS} \
HOSTCC=${HOSTCC:-c99} CC=${CC} LDFLAGS=${LDFLAGS} \
./configure --host=${HOST} --build=$(./config/config.guess) --enable-docs=no --enable-shared=no --enable-static=yes --prefix=/ --with-oniguruma=${cwd}/ios/onig/${arch} $(test -z ${BISON+x} || echo '--enable-maintainer-mode')
make -j${MAKEJOBS} install DESTDIR="${cwd}/ios/jq/${arch}"
make clean
Expand Down
6 changes: 6 additions & 0 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@ AM_PROG_CC_C_O

AC_SYS_LARGEFILE # issue 2167

AC_ARG_VAR([HOSTCC], [Host compiler; necessary to build gen_jq_builtins when cross-compiling])
AC_ARG_VAR([HOSTCFLAGS], [Host compiler flags])
HOSTCC=${HOSTCC:-$CC}
AC_DEFINE([HOSTCC], ["$HOSTCC"])
AC_DEFINE([HOSTCC], ["$HOSTCFLAGS"])

dnl couldn't use AM_PROG_LEX as it doesn't support header files like the
dnl AC_PROG_YACC macros...

Expand Down
2 changes: 1 addition & 1 deletion scripts/crosscompile
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ esac
rm -rf "$plat/tmp"
mkdir "$plat/tmp"
cd "$plat/tmp"
../../../configure "$@"
../../../configure "HOSTCC=${HOSTCC:-c99}" "$@"
make "$jobs" DESTDIR=$plat install
set -x
for jq in `find . -type f \( -name jq -o -name jq.exe \) -print`; do
Expand Down
73 changes: 1 addition & 72 deletions src/builtin.c
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ void *alloca (size_t);
#include "bytecode.h"
#include "linker.h"
#include "locfile.h"
#include "math-macos.h"
#include "jv_unicode.h"
#include "jv_alloc.h"
#include "jv_private.h"
Expand Down Expand Up @@ -113,39 +114,6 @@ jv binop_plus(jv a, jv b) {
}
}

#ifdef __APPLE__
// macOS has a bunch of libm deprecation warnings, so let's clean those up
#ifdef HAVE_TGAMMA
#define HAVE_GAMMA
#define gamma tgamma
#endif
#ifdef HAVE___EXP10
#define HAVE_EXP10
#define exp10 __exp10
#endif
#ifdef HAVE_REMAINDER
#define HAVE_DREM
#define drem remainder
#endif

// We replace significand with our own, since there's not a rename-replacement
#ifdef HAVE_FREXP
static double __jq_significand(double x) {
int z;
return 2*frexp(x, &z);
}
#define HAVE_SIGNIFICAND
#define significand __jq_significand
#elif defined(HAVE_SCALBN) && defined(HAVE_ILOGB)
static double __jq_significand(double x) {
return scalbn(x, -ilogb(x));
}
#define HAVE_SIGNIFICAND
#define significand __jq_significand
#endif

#endif // ifdef __APPLE__

#define LIBM_DD(name) \
static jv f_ ## name(jq_state *jq, jv input) { \
if (jv_get_kind(input) != JV_KIND_NUMBER) { \
Expand Down Expand Up @@ -208,13 +176,6 @@ static jv f_ ## name(jq_state *jq, jv input, jv a, jv b, jv c) { \
#undef LIBM_DDD
#undef LIBM_DD

#ifdef __APPLE__
#undef gamma
#undef drem
#undef significand
#undef exp10
#endif

#ifdef HAVE_FREXP
static jv f_frexp(jq_state *jq, jv input) {
if (jv_get_kind(input) != JV_KIND_NUMBER) {
Expand Down Expand Up @@ -1837,40 +1798,8 @@ static block bind_bytecoded_builtins(block b) {
static const char jq_builtins[] =
/* Include jq-coded builtins */
#include "src/builtin.inc"

/* Include unsupported math functions next */
#define LIBM_DD(name)
#define LIBM_DDD(name)
#define LIBM_DDDD(name)
#define LIBM_DD_NO(name) "def " #name ": \"Error: " #name "/0 not found at build time\"|error;"
#define LIBM_DDD_NO(name) "def " #name "(a;b): \"Error: " #name "/2 not found at build time\"|error;"
#define LIBM_DDDD_NO(name) "def " #name "(a;b;c): \"Error: " #name "/3 not found at build time\"|error;"
#include "libm.h"
#ifndef HAVE_FREXP
"def frexp: \"Error: frexp/0 not found at build time\"|error;"
#endif
#ifndef HAVE_MODF
"def modf: \"Error: modf/0 not found at build time\"|error;"
#endif
#ifndef HAVE_LGAMMA_R
"def lgamma_r: \"Error: lgamma_r/0 not found at build time\"|error;"
#endif
;

#undef LIBM_DDDD_NO
#undef LIBM_DDD_NO
#undef LIBM_DD_NO
#undef LIBM_DDDD
#undef LIBM_DDD
#undef LIBM_DD

#ifdef __APPLE__
#undef HAVE_GAMMA
#undef HAVE_EXP10
#undef HAVE_DREM
#undef HAVE_SIGNIFICAND
#endif

static block gen_builtin_list(block builtins) {
jv list = jv_array_append(block_list_funcs(builtins, 1), jv_string("builtins/0"));
return BLOCK(builtins, gen_function("builtins", gen_noop(), gen_const(list)));
Expand Down
132 changes: 132 additions & 0 deletions src/gen_jq_builtins.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>

#include "main-win32.h"
#include "math-macos.h"

static uint_least8_t index = 0;

static bool indentation(void) {
if (index == 0 && fputs(" ", stdout) == EOF) {
perror("fputs");
return false;
}
return true;
}

static bool separator(void) {
if (putchar(index < 10 ? ' ' : '\n') == EOF) {
perror("putchar");
return false;
}
index = (index + 1) % 11;
return true;
}

static bool gen_char(char const ch) {
if (!indentation())
return false;
if (printf("%.4o,", (unsigned)ch) == EOF)
return false;
return separator();
}

static bool gen_string(char const *const string)
{
for (char const *ch = string; *ch; ++ch) {
if (!gen_char(*ch))
return false;
}
return true;
}

static bool gen_file(FILE *const fp) {
for (int ch; (ch = getc(fp)) != EOF;) {
if (!gen_char(ch))
return false;
}
if (ferror(fp)) {
perror("getc");
return false;
}
return true;
}

DEFINE_MAIN(int argc, char *const argv[]) {
/* argv[1] must be the path to "src/builtin.jq" */
if (argc != 2) {
static char const err[] =
"gen_builtin_inc: Wrong number of arguments.\n";
if (fputs(err, stderr) == EOF)
perror("fputs");
return EXIT_FAILURE;
}

if (puts("{") == EOF) {
perror("puts");
return EXIT_FAILURE;
}

FILE *const builtin_jq = fopen(argv[1], "r");
if (!builtin_jq) {
perror("fopen");
return EXIT_FAILURE;
}
if (!gen_file(builtin_jq))
return EXIT_FAILURE;

#define GEN_STRING(string) \
do { \
if (!gen_string(string)) \
return EXIT_FAILURE; \
} while (0)

/* Unsupported math functions */
#define LIBM_DD(name)
#define LIBM_DDD(name)
#define LIBM_DDDD(name)
#define LIBM_DD_NO(name) \
gen_string( \
"def " #name ":" \
"\"Error: " #name "/0 not found at build time\"|error;");
#define LIBM_DDD_NO(name) \
gen_string( \
"def " #name "(a;b):" \
"\"Error: " #name "/2 not found at build time\"|error;");
#define LIBM_DDDD_NO(name) \
gen_string("def " #name "(a;b;c):" \
"\"Error: " #name "/3 not found at build time\"|error;");
#include "libm.h"
#undef LIBM_DD
#undef LIBM_DDD
#undef LIBM_DDDD
#undef LIBM_DD_NO
#undef LIBM_DDD_NO
#undef LIBM_DDDD_NO

#ifndef HAVE_FREXP
GEN_STRING(
"def frexp:"
"\"Error: frexp/0 not found at build time\"|error;");
#endif
#ifndef HAVE_MODF
GEN_STRING(
"def modf:"
"\"Error: modf/0 not found at build time\"|error;");
#endif
#ifndef HAVE_LGAMMA_R
GEN_STRING(
"def lgamma_r:"
"\"Error: lgamma_r/0 not found at build time\"|error;");
#endif

#undef GEN_STRING

if (puts("'\\0',\n}") == EOF) {
perror("puts");
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
28 changes: 28 additions & 0 deletions src/main-win32.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#ifndef MAIN_WIN32_H
#define MAIN_WIN32_H

#ifdef WIN32
#include <stringapiset.h>
#include <wchar.h>

#define DEFINE_MAIN(ARGC, ARGV) \
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Haven't notice this code before. Is this used when building native for windows, not via emulation layers like msys2 or cygwin?

Copy link
Member Author

@emanuele6 emanuele6 Dec 12, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think, when you compile with -municode on windows, wmain() is called instead of main() as entry point with wchar_t* strings instead of char* strings; then that code in wmain() converts the wchar_t* strings to char* strings encoding with utf-8.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume it is probably an hack to make the program get the correct utf-8 encoding on unicode characters you pass through cmd.exe because windows encodings are weird.
For example, recently, I added a test to curl that verifies that utf-8 bytes get correctly encoded to JSON; and I had to modify my test to get the utf-8 string from a file instead of from command line arguemnts because the windows CI was not passing the right bytes to curl. :/
curl/curl#12434

int umain(ARGC, ARGV); \
int wmain(int argc, wchar_t* wargv[]) { \
size_t arg_sz; \
char **argv = alloca(argc * sizeof(wchar_t*)); \
for (int i = 0; i < argc; i++) { \
argv[i] = alloca((arg_sz = WideCharToMultiByte(CP_UTF8, \
0, \
wargv[i], \
-1, 0, 0, 0, 0))); \
WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, argv[i], arg_sz, 0, 0); \
} \
return umain(argc, argv); \
} \
int umain(ARGC, ARGV)
#else
#define DEFINE_MAIN(ARGC, ARGV) \
int main(ARGC, ARGV)
#endif

#endif
Loading