Skip to content

Commit 3c4fe23

Browse files
indygregindygreg
authored
unix: upgrade libedit 20210910-3.1 -> 20240808-3.1 (#466)
We were soft blocked on upgrading due to musl compatibility issues. It looks like these got fixed upstream. So we refreshed the configure patch and libedit build _just worked_. However, Python 3.9 and 3.10 encountered compile errors with the newer version. On 3.10 we worked around this bug by backporting a patch from 3.11. On 3.9, the backport was non-trivial, so I just hacked up the existing 3.9 patch to manually change some C preprocessor checks to key off libedit. While diffing `Modules/readline.c` I found another patch related to fixing completer delims. While strictly not required, it was trivial to backport to 3.10 to fix some missing functionality. So I did. 3.13 initially didn't like the upgraded libedit because we were manually defining a preprocessor variable (introduced in 3.13 by upstream commit 8515fd79fef1ac16d7848cec5ec1797294cb5366). Removing the variable and letting configure deduce things with the newer libedit appears to _just work_. Perhaps upstream configure doesn't implement the feature detection properly on older libedit versions?
1 parent 19c27b3 commit 3c4fe23

7 files changed

+204
-39
lines changed

‎cpython-unix/build-cpython.sh

+26-2
Original file line numberDiff line numberDiff line change
@@ -221,19 +221,43 @@ if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_9}" ]; then
221221
patch -p1 -i ${ROOT}/patch-decimal-modern-mpdecimal.patch
222222
fi
223223

224+
# We build against libedit instead of readline in all environments.
225+
#
226+
# On macOS, we use the system/SDK libedit, which is likely somewhat old.
227+
#
228+
# On Linux, we use our own libedit, which should be modern.
229+
#
224230
# CPython 3.10 added proper support for building against libedit outside of
225-
# macOS. On older versions, we need to patch readline.c.
226-
if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_9}" ]; then
231+
# macOS. On older versions, we need to hack up readline.c to build against
232+
# libedit. This patch breaks older libedit (as seen on macOS) so don't apply
233+
# on macOS.
234+
if [[ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_9}" && "${PYBUILD_PLATFORM}" != "macos" ]]; then
227235
# readline.c assumes that a modern readline API version has a free_history_entry().
228236
# but libedit does not. Change the #ifdef accordingly.
229237
#
230238
# Similarly, we invoke configure using readline, which sets
231239
# HAVE_RL_COMPLETION_SUPPRESS_APPEND improperly. So hack that. This is a bug
232240
# in our build system, as we should probably be invoking configure again when
233241
# using libedit.
242+
#
243+
# Similar workaround for on_completion_display_matches_hook.
234244
patch -p1 -i ${ROOT}/patch-readline-libedit.patch
235245
fi
236246

247+
if [ "${PYTHON_MAJMIN_VERSION}" = "3.10" ]; then
248+
# Even though 3.10 is libedit aware, it isn't compatible with newer
249+
# versions of libedit. We need to backport a 3.11 patch to teach the
250+
# build system about completions.
251+
# Backport of 9e9df93ffc6df5141843caf651d33d446676a414 from 3.11.
252+
patch -p1 -i ${ROOT}/patch-readline-libedit-completions.patch
253+
254+
# 3.11 has a patch related to completer delims that closes a feature
255+
# gap. Backport it as a quality of life enhancement.
256+
#
257+
# Backport of 42dd2613fe4bc61e1f633078560f2d84a0a16c3f from 3.11.
258+
patch -p1 -i ${ROOT}/patch-readline-libedit-completer-delims.patch
259+
fi
260+
237261
# iOS doesn't have system(). Teach posixmodule.c about that.
238262
# Python 3.11 makes this a configure time check, so we don't need the patch there.
239263
if [[ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_10}" ]]; then

‎cpython-unix/build-libedit.sh

+27-25
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,13 @@ pushd libedit-${LIBEDIT_VERSION}
1919
# run-time. So we hack up the configure script instead.
2020
patch -p1 << "EOF"
2121
diff --git a/configure b/configure
22-
index 26dd8d0..4b6d47c 100755
22+
index 614795f..4671f1b 100755
2323
--- a/configure
2424
+++ b/configure
25-
@@ -12921,14 +12921,14 @@ test -n "$NROFF" || NROFF="/bin/false"
26-
27-
28-
25+
@@ -14154,14 +14154,14 @@ test -n "$NROFF" || NROFF="/bin/false"
26+
27+
28+
2929
-{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for tgetent in -lncurses" >&5
3030
-printf %s "checking for tgetent in -lncurses... " >&6; }
3131
-if test ${ac_cv_lib_ncurses_tgetent+y}
@@ -34,26 +34,28 @@ index 26dd8d0..4b6d47c 100755
3434
+if test ${ac_cv_lib_ncursesw_tgetent+y}
3535
then :
3636
printf %s "(cached) " >&6
37-
else $as_nop
38-
ac_check_lib_save_LIBS=$LIBS
37+
else case e in #(
38+
e) ac_check_lib_save_LIBS=$LIBS
3939
-LIBS="-lncurses $LIBS"
4040
+LIBS="-lncursesw $LIBS"
4141
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
4242
/* end confdefs.h. */
43-
44-
@@ -12946,21 +12946,21 @@ return tgetent ();
43+
44+
@@ -14185,9 +14185,9 @@ return tgetent ();
4545
_ACEOF
4646
if ac_fn_c_try_link "$LINENO"
4747
then :
4848
- ac_cv_lib_ncurses_tgetent=yes
4949
+ ac_cv_lib_ncursesw_tgetent=yes
50-
else $as_nop
51-
- ac_cv_lib_ncurses_tgetent=no
52-
+ ac_cv_lib_ncursesw_tgetent=no
50+
else case e in #(
51+
- e) ac_cv_lib_ncurses_tgetent=no ;;
52+
+ e) ac_cv_lib_ncursesw_tgetent=no ;;
53+
esac
5354
fi
5455
rm -f core conftest.err conftest.$ac_objext conftest.beam \
55-
conftest$ac_exeext conftest.$ac_ext
56-
LIBS=$ac_check_lib_save_LIBS
56+
@@ -14195,13 +14195,13 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \
57+
LIBS=$ac_check_lib_save_LIBS ;;
58+
esac
5759
fi
5860
-{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncurses_tgetent" >&5
5961
-printf "%s\n" "$ac_cv_lib_ncurses_tgetent" >&6; }
@@ -63,21 +65,21 @@ index 26dd8d0..4b6d47c 100755
6365
+if test "x$ac_cv_lib_ncursesw_tgetent" = xyes
6466
then :
6567
printf "%s\n" "#define HAVE_LIBNCURSES 1" >>confdefs.h
66-
68+
6769
- LIBS="-lncurses $LIBS"
6870
+ LIBS="-lncursesw $LIBS"
69-
70-
else $as_nop
71-
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for tgetent in -lcurses" >&5
72-
@@ -13089,7 +13089,7 @@ then :
71+
72+
else case e in #(
73+
e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for tgetent in -lcurses" >&5
74+
@@ -14354,7 +14354,7 @@ then :
7375
LIBS="-ltinfo $LIBS"
74-
75-
else $as_nop
76-
- as_fn_error $? "libncurses, libcurses, libtermcap or libtinfo is required!" "$LINENO" 5
77-
+ as_fn_error $? "libncursesw, libcurses, libtermcap or libtinfo is required!" "$LINENO" 5
78-
76+
77+
else case e in #(
78+
- e) as_fn_error $? "libncurses, libcurses, libtermcap or libtinfo is required!" "$LINENO" 5
79+
+ e) as_fn_error $? "libncursesw, libcurses, libtermcap or libtinfo is required!" "$LINENO" 5
80+
;;
81+
esac
7982
fi
80-
8183
EOF
8284

8385
cflags="${EXTRA_TARGET_CFLAGS} -fPIC -I${TOOLS_PATH}/deps/include -I${TOOLS_PATH}/deps/include/ncursesw"

‎cpython-unix/extension-modules.yml

-2
Original file line numberDiff line numberDiff line change
@@ -880,8 +880,6 @@ readline:
880880
- readline.c
881881
defines:
882882
- USE_LIBEDIT=1
883-
# While some versions do not, our readline `on_startup_hook` takes arguments.
884-
- Py_RL_STARTUP_HOOK_TAKES_ARGS
885883
includes-deps:
886884
- libedit/include
887885
- libedit/include/ncursesw

‎cpython-unix/patch-readline-libedit-completer-delims.patch

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py
2+
index 835280f2281..6c2726d3209 100644
3+
--- a/Lib/test/test_readline.py
4+
+++ b/Lib/test/test_readline.py
5+
@@ -5,6 +5,7 @@
6+
import os
7+
import sys
8+
import tempfile
9+
+import textwrap
10+
import unittest
11+
from test.support import verbose
12+
from test.support.import_helper import import_module
13+
@@ -163,6 +164,25 @@ def test_auto_history_disabled(self):
14+
# end, so don't expect it in the output.
15+
self.assertIn(b"History length: 0", output)
16+
17+
+ def test_set_complete_delims(self):
18+
+ script = textwrap.dedent("""
19+
+ import readline
20+
+ def complete(text, state):
21+
+ if state == 0 and text == "$":
22+
+ return "$complete"
23+
+ return None
24+
+ if "libedit" in getattr(readline, "__doc__", ""):
25+
+ readline.parse_and_bind(r'bind "\\t" rl_complete')
26+
+ else:
27+
+ readline.parse_and_bind(r'"\\t": complete')
28+
+ readline.set_completer_delims(" \\t\\n")
29+
+ readline.set_completer(complete)
30+
+ print(input())
31+
+ """)
32+
+
33+
+ output = run_pty(script, input=b"$\t\n")
34+
+ self.assertIn(b"$complete", output)
35+
+
36+
def test_nonascii(self):
37+
loc = locale.setlocale(locale.LC_CTYPE, None)
38+
if loc in ('C', 'POSIX'):
39+
diff --git a/Modules/readline.c b/Modules/readline.c
40+
index 8c7f526d418..1e13a0e6e06 100644
41+
--- a/Modules/readline.c
42+
+++ b/Modules/readline.c
43+
@@ -572,6 +572,13 @@ readline_set_completer_delims(PyObject *module, PyObject *string)
44+
if (break_chars) {
45+
free(completer_word_break_characters);
46+
completer_word_break_characters = break_chars;
47+
+#ifdef WITH_EDITLINE
48+
+ rl_basic_word_break_characters = break_chars;
49+
+#else
50+
+ if (using_libedit_emulation) {
51+
+ rl_basic_word_break_characters = break_chars;
52+
+ }
53+
+#endif
54+
rl_completer_word_break_characters = break_chars;
55+
Py_RETURN_NONE;
56+
}
57+
@@ -1260,6 +1267,15 @@ setup_readline(readlinestate *mod_state)
58+
completer_word_break_characters =
59+
strdup(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?");
60+
/* All nonalphanums except '.' */
61+
+#ifdef WITH_EDITLINE
62+
+ // libedit uses rl_basic_word_break_characters instead of
63+
+ // rl_completer_word_break_characters as complete delimiter
64+
+ rl_basic_word_break_characters = completer_word_break_characters;
65+
+#else
66+
+ if (using_libedit_emulation) {
67+
+ rl_basic_word_break_characters = completer_word_break_characters;
68+
+ }
69+
+#endif
70+
rl_completer_word_break_characters = completer_word_break_characters;
71+
72+
mod_state->begidx = PyLong_FromLong(0L);

‎cpython-unix/patch-readline-libedit-completions.patch

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
diff --git a/Modules/readline.c b/Modules/readline.c
2+
index 27b89de7279..8c7f526d418 100644
3+
--- a/Modules/readline.c
4+
+++ b/Modules/readline.c
5+
@@ -440,7 +440,7 @@ readline_set_completion_display_matches_hook_impl(PyObject *module,
6+
default completion display. */
7+
rl_completion_display_matches_hook =
8+
readlinestate_global->completion_display_matches_hook ?
9+
-#if defined(_RL_FUNCTION_TYPEDEF)
10+
+#if defined(HAVE_RL_COMPDISP_FUNC_T)
11+
(rl_compdisp_func_t *)on_completion_display_matches_hook : 0;
12+
#else
13+
(VFunction *)on_completion_display_matches_hook : 0;
14+
diff --git a/configure.ac b/configure.ac
15+
index e1cbb7c7fbe..629b7b76c3c 100644
16+
--- a/configure.ac
17+
+++ b/configure.ac
18+
@@ -5918,6 +5918,20 @@ if test "$py_cv_lib_readline" = yes; then
19+
AC_CHECK_LIB($LIBREADLINE, append_history,
20+
AC_DEFINE(HAVE_RL_APPEND_HISTORY, 1,
21+
[Define if readline supports append_history]),,$READLINE_LIBS)
22+
+
23+
+ # in readline as well as newer editline (April 2023)
24+
+ AC_CHECK_TYPE([rl_compdisp_func_t],
25+
+ [AC_DEFINE([HAVE_RL_COMPDISP_FUNC_T], [1],
26+
+ [Define if readline supports rl_compdisp_func_t])],
27+
+ [],
28+
+ [
29+
+#include <stdio.h> /* Must be first for Gnu Readline */
30+
+#ifdef WITH_EDITLINE
31+
+# include <editline/readline.h>
32+
+#else
33+
+# include <readline/readline.h>
34+
+#endif
35+
+ ])
36+
fi
37+
38+
# End of readline checks: restore LIBS
39+
diff --git a/pyconfig.h.in b/pyconfig.h.in
40+
index 0536047f573..94d02e14c44 100644
41+
--- a/pyconfig.h.in
42+
+++ b/pyconfig.h.in
43+
@@ -968,6 +968,9 @@
44+
/* Define if you can turn off readline's signal handling. */
45+
#undef HAVE_RL_CATCH_SIGNAL
46+
47+
+/* Define if readline supports rl_compdisp_func_t */
48+
+#undef HAVE_RL_COMPDISP_FUNC_T
49+
+
50+
/* Define if you have readline 2.2 */
51+
#undef HAVE_RL_COMPLETION_APPEND_CHARACTER
52+

‎cpython-unix/patch-readline-libedit.patch

+23-5
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,32 @@
11
diff --git a/Modules/readline.c b/Modules/readline.c
2-
index 1e74f997b0..56a36e26e6 100644
2+
index 1e74f997b07..0c982857283 100644
33
--- a/Modules/readline.c
44
+++ b/Modules/readline.c
5+
@@ -35,7 +35,7 @@
6+
#define completion_matches(x, y) \
7+
rl_completion_matches((x), ((rl_compentry_func_t *)(y)))
8+
#else
9+
-#if defined(_RL_FUNCTION_TYPEDEF)
10+
+#ifdef USE_LIBEDIT
11+
extern char **completion_matches(char *, rl_compentry_func_t *);
12+
#else
13+
14+
@@ -390,7 +390,7 @@ set_completion_display_matches_hook(PyObject *self, PyObject *args)
15+
default completion display. */
16+
rl_completion_display_matches_hook =
17+
readlinestate_global->completion_display_matches_hook ?
18+
-#if defined(_RL_FUNCTION_TYPEDEF)
19+
+#ifdef USE_LIBEDIT
20+
(rl_compdisp_func_t *)on_completion_display_matches_hook : 0;
21+
#else
22+
(VFunction *)on_completion_display_matches_hook : 0;
523
@@ -511,7 +511,7 @@ set the word delimiters for completion");
6-
24+
725
/* _py_free_history_entry: Utility function to free a history entry. */
8-
26+
927
-#if defined(RL_READLINE_VERSION) && RL_READLINE_VERSION >= 0x0500
1028
+#ifndef USE_LIBEDIT
11-
29+
1230
/* Readline version >= 5.0 introduced a timestamp field into the history entry
1331
structure; this needs to be freed to avoid a memory leak. This version of
1432
@@ -1055,7 +1055,7 @@ flex_complete(const char *text, int start, int end)
@@ -19,7 +37,7 @@ index 1e74f997b0..56a36e26e6 100644
1937
+#ifndef USE_LIBEDIT
2038
rl_completion_suppress_append = 0;
2139
#endif
22-
40+
2341
@@ -1241,7 +1241,7 @@ readline_until_enter_or_signal(const char *prompt, int *signal)
2442
PyEval_SaveThread();
2543
if (s < 0) {

‎pythonbuild/downloads.py

+4-5
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,11 @@
115115
"sha256": "828cb275b91268b1a3ea950d5c0c5eb076c678fdf005d517411f89cc8c3bb416",
116116
"version": "1.0.7",
117117
},
118-
# 20221009-3.1 fails to build on musl due to an includes issue.
119118
"libedit": {
120-
"url": "https://thrysoee.dk/editline/libedit-20210910-3.1.tar.gz",
121-
"size": 524722,
122-
"sha256": "6792a6a992050762edcca28ff3318cdb7de37dccf7bc30db59fcd7017eed13c5",
123-
"version": "20210910-3.1",
119+
"url": "https://thrysoee.dk/editline/libedit-20240808-3.1.tar.gz",
120+
"size": 538611,
121+
"sha256": "5f0573349d77c4a48967191cdd6634dd7aa5f6398c6a57fe037cc02696d6099f",
122+
"version": "20240808-3.1",
124123
"library_names": ["edit"],
125124
"licenses": ["BSD-3-Clause"],
126125
"license_file": "LICENSE.libedit.txt",

0 commit comments

Comments
 (0)