[Git][ghc/ghc][wip/fendor/dep-json-with-options] 6 commits: Refactor Makefile to use phases appropriately
by Hannes Siebenhandl (@fendor) 13 Oct '26
by Hannes Siebenhandl (@fendor) 13 Oct '26
13 Oct '26
Hannes Siebenhandl pushed to branch wip/fendor/dep-json-with-options at Glasgow Haskell Compiler / GHC
Commits:
f1d67d13 by fendor at 2026-10-13T14:35:22+02:00
Refactor Makefile to use phases appropriately
Prepare refactoring to allow json and Makefile implementation
- - - - -
e3f3a4fb by fendor at 2026-10-13T14:35:46+02:00
Add json output format
- - - - -
681f327f by fendor at 2026-10-13T14:35:47+02:00
Add support for multiple home units
- - - - -
0b2d4bb5 by fendor at 2026-10-13T14:35:47+02:00
Record OPTIONS_GHC in the `PreprocessingNode`
This allows user to treat modules if they use -XTemplateHaskell, for
example.
- - - - -
41054c97 by fendor at 2026-10-13T14:35:47+02:00
Deprecate doMkDependHS
- - - - -
30bb9f48 by fendor at 2026-10-13T14:35:47+02:00
Testsuite changes
- - - - -
64 changed files:
- + changelog.d/make-depends-json
- compiler/GHC/Driver/DynFlags.hs
- compiler/GHC/Driver/MakeFile.hs
- compiler/GHC/Driver/Pipeline.hs
- compiler/GHC/Driver/Pipeline/Execute.hs
- compiler/GHC/Driver/Pipeline/Phases.hs
- compiler/GHC/Driver/Session.hs
- compiler/GHC/Driver/Session/Units.hs
- compiler/GHC/Utils/Json.hs
- + docs/users_guide/make-depends-json-schema-1_0.json
- docs/users_guide/separate_compilation.rst
- ghc/Main.hs
- testsuite/driver/testglobals.py
- testsuite/driver/testlib.py
- testsuite/tests/driver/mkdepend/Makefile
- + testsuite/tests/driver/mkdepend/README.md
- testsuite/tests/driver/mkdepend/A.hs → testsuite/tests/driver/mkdepend/T24384/A.hs
- testsuite/tests/driver/mkdepend/C.hs → testsuite/tests/driver/mkdepend/T24384/C.hs
- testsuite/tests/driver/mkdepend/C.hs-boot → testsuite/tests/driver/mkdepend/T24384/C.hs-boot
- testsuite/tests/driver/mkdepend/D.hs → testsuite/tests/driver/mkdepend/T24384/D.hs
- testsuite/tests/driver/mkdepend/E.hs → testsuite/tests/driver/mkdepend/T24384/E.hs
- testsuite/tests/driver/mkdepend/dep-internal/DepInternal.hs → testsuite/tests/driver/mkdepend/T24384/dep-internal/DepInternal.hs
- testsuite/tests/driver/mkdepend/dep-internal/dep-internal.pkg → testsuite/tests/driver/mkdepend/T24384/dep-internal/dep-internal.pkg
- testsuite/tests/driver/mkdepend/dep-public/DepPublic.hs → testsuite/tests/driver/mkdepend/T24384/dep-public/DepPublic.hs
- testsuite/tests/driver/mkdepend/dep-public/dep-public.pkg → testsuite/tests/driver/mkdepend/T24384/dep-public/dep-public.pkg
- testsuite/tests/driver/mkdepend/dep/Dep.hs → testsuite/tests/driver/mkdepend/T24384/dep/Dep.hs
- testsuite/tests/driver/mkdepend/dep/dep.pkg → testsuite/tests/driver/mkdepend/T24384/dep/dep.pkg
- testsuite/tests/driver/mkdepend/foo.h → testsuite/tests/driver/mkdepend/T24384/foo.h
- testsuite/tests/driver/mkdepend/lib/B.hs → testsuite/tests/driver/mkdepend/T24384/lib/B.hs
- + testsuite/tests/driver/mkdepend/T24384/lib/B.hs-boot
- testsuite/tests/driver/mkdepend/preproc.sh → testsuite/tests/driver/mkdepend/T24384/preproc.sh
- testsuite/tests/driver/mkdepend/T24384a.stdout
- + testsuite/tests/driver/mkdepend/T24384b.stdout
- + testsuite/tests/driver/mkdepend/T24384c.stdout
- testsuite/tests/driver/mkdepend/all.T
- + testsuite/tests/driver/mkdepend/format_json.sh
- + testsuite/tests/driver/mkdepend/format_makefile.sh
- + testsuite/tests/driver/mkdepend/mhu01/a/A/Base.hs
- + testsuite/tests/driver/mkdepend/mhu01/a/A/Base.hs-boot
- + testsuite/tests/driver/mkdepend/mhu01/a/A/TH.hs
- + testsuite/tests/driver/mkdepend/mhu01/a/Util.hs
- + testsuite/tests/driver/mkdepend/mhu01/b/B/Normal.hs
- + testsuite/tests/driver/mkdepend/mhu01/b/B/Splice.hs
- + testsuite/tests/driver/mkdepend/mhu01/b/Util.hs
- + testsuite/tests/driver/mkdepend/mhu01/c/C.hs
- + testsuite/tests/driver/mkdepend/mhu01/unitA
- + testsuite/tests/driver/mkdepend/mhu01/unitB
- + testsuite/tests/driver/mkdepend/mhu01/unitC
- + testsuite/tests/driver/mkdepend/mhu02/A.hs
- + testsuite/tests/driver/mkdepend/mhu02/B.hs
- + testsuite/tests/driver/mkdepend/mhu02/pp.sh
- + testsuite/tests/driver/mkdepend/mhu02/unitA
- + testsuite/tests/driver/mkdepend/mhu02/unitB
- + testsuite/tests/driver/mkdepend/mhu03/pkg/A.hs
- + testsuite/tests/driver/mkdepend/mhu03/pkg/pp.sh
- + testsuite/tests/driver/mkdepend/mhu03/unitA
- + testsuite/tests/driver/mkdepend/mkdepend01.dep-json
- + testsuite/tests/driver/mkdepend/mkdepend01.dep-makefile
- + testsuite/tests/driver/mkdepend/mkdepend01/A.hs
- + testsuite/tests/driver/mkdepend/mkdepend_mhu01.dep-json
- + testsuite/tests/driver/mkdepend/mkdepend_mhu01.dep-makefile
- + testsuite/tests/driver/mkdepend/mkdepend_mhu02.dep-json
- + testsuite/tests/driver/mkdepend/mkdepend_mhu03.dep-json
- testsuite/tests/perf/compiler/large-project/all.T
The diff was not included because it is too large.
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/ad7bf2eb3fd25ac6e99038d48ee30c…
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/ad7bf2eb3fd25ac6e99038d48ee30c…
You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help
1
0
[Git][ghc/ghc][wip/fendor/dep-json] 6 commits: Refactor Makefile to use phases appropriately
by Hannes Siebenhandl (@fendor) 13 Oct '26
by Hannes Siebenhandl (@fendor) 13 Oct '26
13 Oct '26
Hannes Siebenhandl pushed to branch wip/fendor/dep-json at Glasgow Haskell Compiler / GHC
Commits:
f1d67d13 by fendor at 2026-10-13T14:35:22+02:00
Refactor Makefile to use phases appropriately
Prepare refactoring to allow json and Makefile implementation
- - - - -
e3f3a4fb by fendor at 2026-10-13T14:35:46+02:00
Add json output format
- - - - -
681f327f by fendor at 2026-10-13T14:35:47+02:00
Add support for multiple home units
- - - - -
0b2d4bb5 by fendor at 2026-10-13T14:35:47+02:00
Record OPTIONS_GHC in the `PreprocessingNode`
This allows user to treat modules if they use -XTemplateHaskell, for
example.
- - - - -
41054c97 by fendor at 2026-10-13T14:35:47+02:00
Deprecate doMkDependHS
- - - - -
30bb9f48 by fendor at 2026-10-13T14:35:47+02:00
Testsuite changes
- - - - -
64 changed files:
- + changelog.d/make-depends-json
- compiler/GHC/Driver/DynFlags.hs
- compiler/GHC/Driver/MakeFile.hs
- compiler/GHC/Driver/Pipeline.hs
- compiler/GHC/Driver/Pipeline/Execute.hs
- compiler/GHC/Driver/Pipeline/Phases.hs
- compiler/GHC/Driver/Session.hs
- compiler/GHC/Driver/Session/Units.hs
- compiler/GHC/Utils/Json.hs
- + docs/users_guide/make-depends-json-schema-1_0.json
- docs/users_guide/separate_compilation.rst
- ghc/Main.hs
- testsuite/driver/testglobals.py
- testsuite/driver/testlib.py
- testsuite/tests/driver/mkdepend/Makefile
- + testsuite/tests/driver/mkdepend/README.md
- testsuite/tests/driver/mkdepend/A.hs → testsuite/tests/driver/mkdepend/T24384/A.hs
- testsuite/tests/driver/mkdepend/C.hs → testsuite/tests/driver/mkdepend/T24384/C.hs
- testsuite/tests/driver/mkdepend/C.hs-boot → testsuite/tests/driver/mkdepend/T24384/C.hs-boot
- testsuite/tests/driver/mkdepend/D.hs → testsuite/tests/driver/mkdepend/T24384/D.hs
- testsuite/tests/driver/mkdepend/E.hs → testsuite/tests/driver/mkdepend/T24384/E.hs
- testsuite/tests/driver/mkdepend/dep-internal/DepInternal.hs → testsuite/tests/driver/mkdepend/T24384/dep-internal/DepInternal.hs
- testsuite/tests/driver/mkdepend/dep-internal/dep-internal.pkg → testsuite/tests/driver/mkdepend/T24384/dep-internal/dep-internal.pkg
- testsuite/tests/driver/mkdepend/dep-public/DepPublic.hs → testsuite/tests/driver/mkdepend/T24384/dep-public/DepPublic.hs
- testsuite/tests/driver/mkdepend/dep-public/dep-public.pkg → testsuite/tests/driver/mkdepend/T24384/dep-public/dep-public.pkg
- testsuite/tests/driver/mkdepend/dep/Dep.hs → testsuite/tests/driver/mkdepend/T24384/dep/Dep.hs
- testsuite/tests/driver/mkdepend/dep/dep.pkg → testsuite/tests/driver/mkdepend/T24384/dep/dep.pkg
- testsuite/tests/driver/mkdepend/foo.h → testsuite/tests/driver/mkdepend/T24384/foo.h
- testsuite/tests/driver/mkdepend/lib/B.hs → testsuite/tests/driver/mkdepend/T24384/lib/B.hs
- + testsuite/tests/driver/mkdepend/T24384/lib/B.hs-boot
- testsuite/tests/driver/mkdepend/preproc.sh → testsuite/tests/driver/mkdepend/T24384/preproc.sh
- testsuite/tests/driver/mkdepend/T24384a.stdout
- + testsuite/tests/driver/mkdepend/T24384b.stdout
- + testsuite/tests/driver/mkdepend/T24384c.stdout
- testsuite/tests/driver/mkdepend/all.T
- + testsuite/tests/driver/mkdepend/format_json.sh
- + testsuite/tests/driver/mkdepend/format_makefile.sh
- + testsuite/tests/driver/mkdepend/mhu01/a/A/Base.hs
- + testsuite/tests/driver/mkdepend/mhu01/a/A/Base.hs-boot
- + testsuite/tests/driver/mkdepend/mhu01/a/A/TH.hs
- + testsuite/tests/driver/mkdepend/mhu01/a/Util.hs
- + testsuite/tests/driver/mkdepend/mhu01/b/B/Normal.hs
- + testsuite/tests/driver/mkdepend/mhu01/b/B/Splice.hs
- + testsuite/tests/driver/mkdepend/mhu01/b/Util.hs
- + testsuite/tests/driver/mkdepend/mhu01/c/C.hs
- + testsuite/tests/driver/mkdepend/mhu01/unitA
- + testsuite/tests/driver/mkdepend/mhu01/unitB
- + testsuite/tests/driver/mkdepend/mhu01/unitC
- + testsuite/tests/driver/mkdepend/mhu02/A.hs
- + testsuite/tests/driver/mkdepend/mhu02/B.hs
- + testsuite/tests/driver/mkdepend/mhu02/pp.sh
- + testsuite/tests/driver/mkdepend/mhu02/unitA
- + testsuite/tests/driver/mkdepend/mhu02/unitB
- + testsuite/tests/driver/mkdepend/mhu03/pkg/A.hs
- + testsuite/tests/driver/mkdepend/mhu03/pkg/pp.sh
- + testsuite/tests/driver/mkdepend/mhu03/unitA
- + testsuite/tests/driver/mkdepend/mkdepend01.dep-json
- + testsuite/tests/driver/mkdepend/mkdepend01.dep-makefile
- + testsuite/tests/driver/mkdepend/mkdepend01/A.hs
- + testsuite/tests/driver/mkdepend/mkdepend_mhu01.dep-json
- + testsuite/tests/driver/mkdepend/mkdepend_mhu01.dep-makefile
- + testsuite/tests/driver/mkdepend/mkdepend_mhu02.dep-json
- + testsuite/tests/driver/mkdepend/mkdepend_mhu03.dep-json
- testsuite/tests/perf/compiler/large-project/all.T
The diff was not included because it is too large.
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/888af6d7c15a9455e28bcb419b217e…
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/888af6d7c15a9455e28bcb419b217e…
You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help
1
0
[Git][ghc/ghc] Pushed new branch wip/fendor/dep-json-with-options
by Hannes Siebenhandl (@fendor) 13 Oct '26
by Hannes Siebenhandl (@fendor) 13 Oct '26
13 Oct '26
Hannes Siebenhandl pushed new branch wip/fendor/dep-json-with-options at Glasgow Haskell Compiler / GHC
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/tree/wip/fendor/dep-json-with-options
You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help
1
0
[Git][ghc/ghc][wip/supersven/hadrian-cross-stage3] 12 commits: Simplify the error-suppression mechanism in GHC.Tc.Errors
by Sven Tennie (@supersven) 13 Oct '26
by Sven Tennie (@supersven) 13 Oct '26
13 Oct '26
Sven Tennie pushed to branch wip/supersven/hadrian-cross-stage3 at Glasgow Haskell Compiler / GHC
Commits:
41e80f41 by Simon Peyton Jones at 2026-10-11T14:38:51-04:00
Simplify the error-suppression mechanism in GHC.Tc.Errors
In `GHC.Tc.Errors.reportWanteds` we suppress some errors in favour of
others. But the mechanism had grown crufty, and #27331 exposed a bug.
This MR makes it simpler and more uniform, by doing everything via
the `ei_suppress` field of the ErrorItem.
Claude then found another closely-related bug (see the review linked
on !16564). That bug is immortalised in new test T18851d.
I'm still not happy with `ignoreConstraint` but we can worry about
that another day; I have not touched it.
Main changes
* Instead of /predicting/ when an error will be reported, we set
`cec_error_reported` when an error is /actually/ reported. (I renamed
it from `cec_suppress` to reflect this change.)
Now errors are suppressed only when a preceding error has actually been
reported. Simpler and more robust.
See `Note [cec_error_reported: suppressing less-serious errors]`
The big changes are in `reportWanteds` which looks very different.
* New `Note [ei_suppress: suppressing confusing errors]` describes the
(quite separate) `ei_suppress` mechanism for suppressing confusing
errors, mainly arising from non-empty rewriter sets
* In `tryTypeClassDefaulting`, instead of switching off type-class defaulting
if there is an insoluble /anywhere/, we just switch it off locally when
we find an insoluble sub-tree. See Note [When to do type-class defaulting]
and (W5) in Note [ApproximateWC]
* Greatly simplify `insolubleWantedCt`.
* I made type-class-defaulting be a bit more eager, only switched off when
a peer or ancestor (but not child) constraint is insoluble. See
Note [When to do type-class defaulting].
Small refactorings
* Rename `askNoErrs` to `askErrsFound` and flip its polarity
* The code for "check if ei_suppress is true of all items" code is
moved to `mkErrorItems` rather than in the large `reportWanteds`.
* Code for `has_gadt_match` moves to `mkGivenErrorReporter`, rather than
polluting the large `reportWanteds`.
* Many tests with `-fdefer-type-errors` get more warnings, especially
about ambiguous (Num a) constraints. This is by-design: see
Note [Always warn with -fdefer-type-errors]. Previously GHC was not
honouring this Note but now it is, hence the extra errors.
- - - - -
f217fc31 by Cheng Shao at 2026-10-11T14:39:33-04:00
compiler: ensure cross ghc --info output target unit ids
Previously, for cross ghcs, the base/ghc/ghc-internal unit ids in
--info output do not match the target unit ids reported by ghc-pkg,
breaking cross ghc release builds with hash-unit-ids when being used
with cabal 3.18. This patch fixes it by making --info output target
unit ids. Fixes #27919.
Co-authored-by: Codex <codex(a)openai.com>
- - - - -
84ac2475 by Sven Tennie at 2026-10-13T10:56:52+02:00
hadrian: Add Stage3 cross-compiled target bindist (#26924)
Add a `binary-dist-stage3` Hadrian target that packages target
executables (produced by the stage2 compiler) into a separate
'_build/bindist-stage3/' folder, distinct from the stage2 regular or
cross-compiler bindist in '_build/bindist/'.
This allows a single CI pipeline job to produce both a cross-compiler
bindist (e.g. x86_64 -> RISC-V) and a target-architecture bindist (e.g.
RISC-V -> RISC-V) that can be installed and run natively on the target.
To avoid issues with stale files or race-conditions on them, generate
the `configure` script per stage in `_build/<stage>/distrib`
directories.
- - - - -
9792dde4 by Sven Tennie at 2026-10-13T10:56:53+02:00
ci: Build Stage3 and Stage2 bindists for cross targets in one job (#26924)
Building the stage3 target bindist already produces most of the stage2
cross-compiler bindist as a byproduct, so building them in separate jobs
duplicates the build efforts for no benefit. Instead of a separate
CROSS_STAGE=3 job, the stage3 job now also builds and publishes the
stage2 bindist.
The stage3 tarball is named and versioned as if it were built natively
on the target (target triple prefix, non-cross opsys), so its artifact
name stays what downstream consumers already expect and no changes are
needed on their side.
For now, this is only enabled for the RISC-V job. Others can easily
follow.
- - - - -
936a20e0 by Sven Tennie at 2026-10-13T10:56:53+02:00
ci: model cross targets as structured TargetPlatform triples
Storing the cross target as a raw triple forced ad-hoc parsing at every
use, and broke the single responsibility rule. Modelling it as a
structured `TargetPlatform`` instead, avoids parsing and provides a
clear separation of concerns.
- - - - -
39b63d0a by Sven Tennie at 2026-10-13T10:56:53+02:00
ci: use a scalar cross-stage switch, FINAL_CROSS_STAGE=2|3
For cross-compilers, Stage2 bindists are (almost) a by-product of Stage3
bindists. Thus, we either build Stage2 or Stage2 + Stage3 bindists. The
commit at hand reflects this fact in types.
- - - - -
c0b952a4 by Sven Tennie at 2026-10-13T10:56:53+02:00
ci: extract the cross smoke tests into a separate smoke_test command
This reduces if-nesting and complexity of the main `test_hadrian`
command. The decision how to test is now made by the caller. This is a
fair choice, because most test cases can run under emulation - they are
just very slow then.
- - - - -
52f40f7e by Sven Tennie at 2026-10-13T10:56:53+02:00
ci: inline the FINAL_CROSS_STAGE renderer
`finalCrossStageEnvVal` was small and only used in one place.
- - - - -
3c0c55ec by Sven Tennie at 2026-10-13T10:56:53+02:00
hadrian: clarify cross-stage settings and project.mk comments
Reword the settings-rule comment to name the two stage roles
(compilerStage vs libraryStage) and explain what 'the final
architecture' in terms of project.mk generation means.
- - - - -
52256473 by Sven Tennie at 2026-10-13T11:01:18+02:00
Fix intepreter settings for native target bindists
Stage3 cross-compiled bindists wrongly assumed they cannot run the
interpreter natively. Fix this by adjusting compile flags and choosing
the correct libs/target/default.target files per compiler stage.
- - - - -
be6eeb1b by Sven Tennie at 2026-10-13T11:01:18+02:00
Generate default.target only from the successor stage for stage0
This mirrors the behaviour for the settings' rules. Reflects that Stage0
compilers depend on the Stage1 lib/ dir.
- - - - -
c64ae620 by GHC GitLab CI at 2026-10-13T12:14:47+02:00
Reduce libraryStage equation duplication
The same code was duplicated in a couple of places.
- - - - -
71 changed files:
- .gitignore
- .gitlab/ci.sh
- .gitlab/generate-ci/gen_ci.hs
- .gitlab/hello.hs
- .gitlab/jobs.yaml
- + changelog.d/T27731
- + changelog.d/fix-cross-ghc-info-unit-id
- + changelog.d/stage3-cross-bindists
- compiler/GHC/Builtin/WiredIn/Ids.hs
- compiler/GHC/Driver/Session.hs
- compiler/GHC/Rename/Module.hs
- compiler/GHC/Settings.hs
- compiler/GHC/Settings/IO.hs
- compiler/GHC/Tc/Errors.hs
- compiler/GHC/Tc/Errors/Ppr.hs
- compiler/GHC/Tc/Errors/Types.hs
- compiler/GHC/Tc/Instance/Family.hs
- compiler/GHC/Tc/Solver/Default.hs
- compiler/GHC/Tc/Solver/Equality.hs
- compiler/GHC/Tc/Types/Constraint.hs
- compiler/GHC/Tc/Utils/Monad.hs
- distrib/configure.ac.in
- hadrian/bindist/Makefile
- hadrian/bindist/config.mk.in
- hadrian/src/BindistConfig.hs
- hadrian/src/Oracles/Setting.hs
- hadrian/src/Rules/BinaryDist.hs
- hadrian/src/Rules/Generate.hs
- hadrian/src/Settings/Packages.hs
- testsuite/tests/ghci/prog020/ghci.prog020a.stderr
- testsuite/tests/ghci/prog020/ghci.prog020b.stderr
- testsuite/tests/ghci/prog021/prog021a.stderr
- testsuite/tests/ghci/prog021/prog021b.stderr
- testsuite/tests/ghci/prog022/ghci.prog022a.stderr
- testsuite/tests/ghci/prog022/ghci.prog022b.stderr
- testsuite/tests/ghci/prog022/ghci.prog022c.stderr
- testsuite/tests/ghci/prog022/ghci.prog022d.stderr
- testsuite/tests/ghci/prog022/ghci.prog022e.stderr
- testsuite/tests/ghci/prog022/ghci.prog022f.stderr
- testsuite/tests/ghci/scripts/Defer02.stderr
- testsuite/tests/module/mod130.stderr
- testsuite/tests/overloadedstrings/should_fail/T25926.stderr
- testsuite/tests/overloadedstrings/should_fail/T27124.stderr
- testsuite/tests/patsyn/should_fail/records-nofieldselectors.stderr
- testsuite/tests/printer/T17697.stderr
- testsuite/tests/rename/should_fail/T15607.stderr
- testsuite/tests/rename/should_fail/T19843b.stderr
- testsuite/tests/safeHaskell/ghci/p4.stderr
- testsuite/tests/safeHaskell/ghci/p6.stderr
- testsuite/tests/typecheck/should_compile/T11254.stderr
- testsuite/tests/typecheck/should_compile/T15370.stderr
- testsuite/tests/typecheck/should_compile/valid_hole_fits.stderr
- testsuite/tests/typecheck/should_fail/ClassOperator.stderr
- testsuite/tests/typecheck/should_fail/DoExpansion2.stderr
- testsuite/tests/typecheck/should_fail/DoExpansion3.stderr
- testsuite/tests/typecheck/should_fail/FunDepOrigin1b.hs
- testsuite/tests/typecheck/should_fail/T18851.stderr
- + testsuite/tests/typecheck/should_fail/T18851e.hs
- + testsuite/tests/typecheck/should_fail/T18851e.stderr
- + testsuite/tests/typecheck/should_fail/T18851f.hs
- + testsuite/tests/typecheck/should_fail/T18851f.stderr
- testsuite/tests/typecheck/should_fail/T22940.stderr
- testsuite/tests/typecheck/should_fail/T26823.stderr
- + testsuite/tests/typecheck/should_fail/T27731.hs
- + testsuite/tests/typecheck/should_fail/T27731.stderr
- testsuite/tests/typecheck/should_fail/T5691.stderr
- testsuite/tests/typecheck/should_fail/T7437.stderr
- testsuite/tests/typecheck/should_fail/all.T
- testsuite/tests/typecheck/should_fail/mc24.stderr
- testsuite/tests/typecheck/should_fail/tcfail019.stderr
- testsuite/tests/typecheck/should_fail/tcfail189.stderr
The diff was not included because it is too large.
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/28789d48f15e05de2ee98413e886b6…
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/28789d48f15e05de2ee98413e886b6…
You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help
1
0
[Git][ghc/ghc][wip/supersven/hadrian-cross-stage3] 12 commits: Simplify the error-suppression mechanism in GHC.Tc.Errors
by Sven Tennie (@supersven) 13 Oct '26
by Sven Tennie (@supersven) 13 Oct '26
13 Oct '26
Sven Tennie pushed to branch wip/supersven/hadrian-cross-stage3 at Glasgow Haskell Compiler / GHC
Commits:
41e80f41 by Simon Peyton Jones at 2026-10-11T14:38:51-04:00
Simplify the error-suppression mechanism in GHC.Tc.Errors
In `GHC.Tc.Errors.reportWanteds` we suppress some errors in favour of
others. But the mechanism had grown crufty, and #27331 exposed a bug.
This MR makes it simpler and more uniform, by doing everything via
the `ei_suppress` field of the ErrorItem.
Claude then found another closely-related bug (see the review linked
on !16564). That bug is immortalised in new test T18851d.
I'm still not happy with `ignoreConstraint` but we can worry about
that another day; I have not touched it.
Main changes
* Instead of /predicting/ when an error will be reported, we set
`cec_error_reported` when an error is /actually/ reported. (I renamed
it from `cec_suppress` to reflect this change.)
Now errors are suppressed only when a preceding error has actually been
reported. Simpler and more robust.
See `Note [cec_error_reported: suppressing less-serious errors]`
The big changes are in `reportWanteds` which looks very different.
* New `Note [ei_suppress: suppressing confusing errors]` describes the
(quite separate) `ei_suppress` mechanism for suppressing confusing
errors, mainly arising from non-empty rewriter sets
* In `tryTypeClassDefaulting`, instead of switching off type-class defaulting
if there is an insoluble /anywhere/, we just switch it off locally when
we find an insoluble sub-tree. See Note [When to do type-class defaulting]
and (W5) in Note [ApproximateWC]
* Greatly simplify `insolubleWantedCt`.
* I made type-class-defaulting be a bit more eager, only switched off when
a peer or ancestor (but not child) constraint is insoluble. See
Note [When to do type-class defaulting].
Small refactorings
* Rename `askNoErrs` to `askErrsFound` and flip its polarity
* The code for "check if ei_suppress is true of all items" code is
moved to `mkErrorItems` rather than in the large `reportWanteds`.
* Code for `has_gadt_match` moves to `mkGivenErrorReporter`, rather than
polluting the large `reportWanteds`.
* Many tests with `-fdefer-type-errors` get more warnings, especially
about ambiguous (Num a) constraints. This is by-design: see
Note [Always warn with -fdefer-type-errors]. Previously GHC was not
honouring this Note but now it is, hence the extra errors.
- - - - -
f217fc31 by Cheng Shao at 2026-10-11T14:39:33-04:00
compiler: ensure cross ghc --info output target unit ids
Previously, for cross ghcs, the base/ghc/ghc-internal unit ids in
--info output do not match the target unit ids reported by ghc-pkg,
breaking cross ghc release builds with hash-unit-ids when being used
with cabal 3.18. This patch fixes it by making --info output target
unit ids. Fixes #27919.
Co-authored-by: Codex <codex(a)openai.com>
- - - - -
84ac2475 by Sven Tennie at 2026-10-13T10:56:52+02:00
hadrian: Add Stage3 cross-compiled target bindist (#26924)
Add a `binary-dist-stage3` Hadrian target that packages target
executables (produced by the stage2 compiler) into a separate
'_build/bindist-stage3/' folder, distinct from the stage2 regular or
cross-compiler bindist in '_build/bindist/'.
This allows a single CI pipeline job to produce both a cross-compiler
bindist (e.g. x86_64 -> RISC-V) and a target-architecture bindist (e.g.
RISC-V -> RISC-V) that can be installed and run natively on the target.
To avoid issues with stale files or race-conditions on them, generate
the `configure` script per stage in `_build/<stage>/distrib`
directories.
- - - - -
9792dde4 by Sven Tennie at 2026-10-13T10:56:53+02:00
ci: Build Stage3 and Stage2 bindists for cross targets in one job (#26924)
Building the stage3 target bindist already produces most of the stage2
cross-compiler bindist as a byproduct, so building them in separate jobs
duplicates the build efforts for no benefit. Instead of a separate
CROSS_STAGE=3 job, the stage3 job now also builds and publishes the
stage2 bindist.
The stage3 tarball is named and versioned as if it were built natively
on the target (target triple prefix, non-cross opsys), so its artifact
name stays what downstream consumers already expect and no changes are
needed on their side.
For now, this is only enabled for the RISC-V job. Others can easily
follow.
- - - - -
936a20e0 by Sven Tennie at 2026-10-13T10:56:53+02:00
ci: model cross targets as structured TargetPlatform triples
Storing the cross target as a raw triple forced ad-hoc parsing at every
use, and broke the single responsibility rule. Modelling it as a
structured `TargetPlatform`` instead, avoids parsing and provides a
clear separation of concerns.
- - - - -
39b63d0a by Sven Tennie at 2026-10-13T10:56:53+02:00
ci: use a scalar cross-stage switch, FINAL_CROSS_STAGE=2|3
For cross-compilers, Stage2 bindists are (almost) a by-product of Stage3
bindists. Thus, we either build Stage2 or Stage2 + Stage3 bindists. The
commit at hand reflects this fact in types.
- - - - -
c0b952a4 by Sven Tennie at 2026-10-13T10:56:53+02:00
ci: extract the cross smoke tests into a separate smoke_test command
This reduces if-nesting and complexity of the main `test_hadrian`
command. The decision how to test is now made by the caller. This is a
fair choice, because most test cases can run under emulation - they are
just very slow then.
- - - - -
52f40f7e by Sven Tennie at 2026-10-13T10:56:53+02:00
ci: inline the FINAL_CROSS_STAGE renderer
`finalCrossStageEnvVal` was small and only used in one place.
- - - - -
3c0c55ec by Sven Tennie at 2026-10-13T10:56:53+02:00
hadrian: clarify cross-stage settings and project.mk comments
Reword the settings-rule comment to name the two stage roles
(compilerStage vs libraryStage) and explain what 'the final
architecture' in terms of project.mk generation means.
- - - - -
52256473 by Sven Tennie at 2026-10-13T11:01:18+02:00
Fix intepreter settings for native target bindists
Stage3 cross-compiled bindists wrongly assumed they cannot run the
interpreter natively. Fix this by adjusting compile flags and choosing
the correct libs/target/default.target files per compiler stage.
- - - - -
be6eeb1b by Sven Tennie at 2026-10-13T11:01:18+02:00
Generate default.target only from the successor stage for stage0
This mirrors the behaviour for the settings' rules. Reflects that Stage0
compilers depend on the Stage1 lib/ dir.
- - - - -
c64ae620 by GHC GitLab CI at 2026-10-13T12:14:47+02:00
Reduce libraryStage equation duplication
The same code was duplicated in a couple of places.
- - - - -
71 changed files:
- .gitignore
- .gitlab/ci.sh
- .gitlab/generate-ci/gen_ci.hs
- .gitlab/hello.hs
- .gitlab/jobs.yaml
- + changelog.d/T27731
- + changelog.d/fix-cross-ghc-info-unit-id
- + changelog.d/stage3-cross-bindists
- compiler/GHC/Builtin/WiredIn/Ids.hs
- compiler/GHC/Driver/Session.hs
- compiler/GHC/Rename/Module.hs
- compiler/GHC/Settings.hs
- compiler/GHC/Settings/IO.hs
- compiler/GHC/Tc/Errors.hs
- compiler/GHC/Tc/Errors/Ppr.hs
- compiler/GHC/Tc/Errors/Types.hs
- compiler/GHC/Tc/Instance/Family.hs
- compiler/GHC/Tc/Solver/Default.hs
- compiler/GHC/Tc/Solver/Equality.hs
- compiler/GHC/Tc/Types/Constraint.hs
- compiler/GHC/Tc/Utils/Monad.hs
- distrib/configure.ac.in
- hadrian/bindist/Makefile
- hadrian/bindist/config.mk.in
- hadrian/src/BindistConfig.hs
- hadrian/src/Oracles/Setting.hs
- hadrian/src/Rules/BinaryDist.hs
- hadrian/src/Rules/Generate.hs
- hadrian/src/Settings/Packages.hs
- testsuite/tests/ghci/prog020/ghci.prog020a.stderr
- testsuite/tests/ghci/prog020/ghci.prog020b.stderr
- testsuite/tests/ghci/prog021/prog021a.stderr
- testsuite/tests/ghci/prog021/prog021b.stderr
- testsuite/tests/ghci/prog022/ghci.prog022a.stderr
- testsuite/tests/ghci/prog022/ghci.prog022b.stderr
- testsuite/tests/ghci/prog022/ghci.prog022c.stderr
- testsuite/tests/ghci/prog022/ghci.prog022d.stderr
- testsuite/tests/ghci/prog022/ghci.prog022e.stderr
- testsuite/tests/ghci/prog022/ghci.prog022f.stderr
- testsuite/tests/ghci/scripts/Defer02.stderr
- testsuite/tests/module/mod130.stderr
- testsuite/tests/overloadedstrings/should_fail/T25926.stderr
- testsuite/tests/overloadedstrings/should_fail/T27124.stderr
- testsuite/tests/patsyn/should_fail/records-nofieldselectors.stderr
- testsuite/tests/printer/T17697.stderr
- testsuite/tests/rename/should_fail/T15607.stderr
- testsuite/tests/rename/should_fail/T19843b.stderr
- testsuite/tests/safeHaskell/ghci/p4.stderr
- testsuite/tests/safeHaskell/ghci/p6.stderr
- testsuite/tests/typecheck/should_compile/T11254.stderr
- testsuite/tests/typecheck/should_compile/T15370.stderr
- testsuite/tests/typecheck/should_compile/valid_hole_fits.stderr
- testsuite/tests/typecheck/should_fail/ClassOperator.stderr
- testsuite/tests/typecheck/should_fail/DoExpansion2.stderr
- testsuite/tests/typecheck/should_fail/DoExpansion3.stderr
- testsuite/tests/typecheck/should_fail/FunDepOrigin1b.hs
- testsuite/tests/typecheck/should_fail/T18851.stderr
- + testsuite/tests/typecheck/should_fail/T18851e.hs
- + testsuite/tests/typecheck/should_fail/T18851e.stderr
- + testsuite/tests/typecheck/should_fail/T18851f.hs
- + testsuite/tests/typecheck/should_fail/T18851f.stderr
- testsuite/tests/typecheck/should_fail/T22940.stderr
- testsuite/tests/typecheck/should_fail/T26823.stderr
- + testsuite/tests/typecheck/should_fail/T27731.hs
- + testsuite/tests/typecheck/should_fail/T27731.stderr
- testsuite/tests/typecheck/should_fail/T5691.stderr
- testsuite/tests/typecheck/should_fail/T7437.stderr
- testsuite/tests/typecheck/should_fail/all.T
- testsuite/tests/typecheck/should_fail/mc24.stderr
- testsuite/tests/typecheck/should_fail/tcfail019.stderr
- testsuite/tests/typecheck/should_fail/tcfail189.stderr
The diff was not included because it is too large.
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/28789d48f15e05de2ee98413e886b6…
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/28789d48f15e05de2ee98413e886b6…
You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help
1
0
[Git][ghc/ghc][wip/fendor/dep-json] 5 commits: Add json output format
by Hannes Siebenhandl (@fendor) 13 Oct '26
by Hannes Siebenhandl (@fendor) 13 Oct '26
13 Oct '26
Hannes Siebenhandl pushed to branch wip/fendor/dep-json at Glasgow Haskell Compiler / GHC
Commits:
39f09c8f by fendor at 2026-10-13T13:35:24+02:00
Add json output format
- - - - -
44ad86e9 by fendor at 2026-10-13T13:35:24+02:00
Add support for multiple home units
- - - - -
aed90a10 by fendor at 2026-10-13T13:35:24+02:00
Record OPTIONS_GHC in the `PreprocessingNode`
This allows user to treat modules if they use -XTemplateHaskell, for
example.
- - - - -
a8d7c8f3 by fendor at 2026-10-13T13:35:24+02:00
Deprecate doMkDependHS
- - - - -
888af6d7 by fendor at 2026-10-13T13:35:24+02:00
Testsuite changes
- - - - -
63 changed files:
- + changelog.d/make-depends-json
- compiler/GHC/Driver/DynFlags.hs
- compiler/GHC/Driver/MakeFile.hs
- compiler/GHC/Driver/Pipeline/Execute.hs
- compiler/GHC/Driver/Pipeline/Phases.hs
- compiler/GHC/Driver/Session.hs
- compiler/GHC/Driver/Session/Units.hs
- compiler/GHC/Utils/Json.hs
- + docs/users_guide/make-depends-json-schema-1_0.json
- docs/users_guide/separate_compilation.rst
- ghc/Main.hs
- testsuite/driver/testglobals.py
- testsuite/driver/testlib.py
- testsuite/tests/driver/mkdepend/Makefile
- + testsuite/tests/driver/mkdepend/README.md
- testsuite/tests/driver/mkdepend/A.hs → testsuite/tests/driver/mkdepend/T24384/A.hs
- testsuite/tests/driver/mkdepend/C.hs → testsuite/tests/driver/mkdepend/T24384/C.hs
- testsuite/tests/driver/mkdepend/C.hs-boot → testsuite/tests/driver/mkdepend/T24384/C.hs-boot
- testsuite/tests/driver/mkdepend/D.hs → testsuite/tests/driver/mkdepend/T24384/D.hs
- testsuite/tests/driver/mkdepend/E.hs → testsuite/tests/driver/mkdepend/T24384/E.hs
- testsuite/tests/driver/mkdepend/dep-internal/DepInternal.hs → testsuite/tests/driver/mkdepend/T24384/dep-internal/DepInternal.hs
- testsuite/tests/driver/mkdepend/dep-internal/dep-internal.pkg → testsuite/tests/driver/mkdepend/T24384/dep-internal/dep-internal.pkg
- testsuite/tests/driver/mkdepend/dep-public/DepPublic.hs → testsuite/tests/driver/mkdepend/T24384/dep-public/DepPublic.hs
- testsuite/tests/driver/mkdepend/dep-public/dep-public.pkg → testsuite/tests/driver/mkdepend/T24384/dep-public/dep-public.pkg
- testsuite/tests/driver/mkdepend/dep/Dep.hs → testsuite/tests/driver/mkdepend/T24384/dep/Dep.hs
- testsuite/tests/driver/mkdepend/dep/dep.pkg → testsuite/tests/driver/mkdepend/T24384/dep/dep.pkg
- testsuite/tests/driver/mkdepend/foo.h → testsuite/tests/driver/mkdepend/T24384/foo.h
- testsuite/tests/driver/mkdepend/lib/B.hs → testsuite/tests/driver/mkdepend/T24384/lib/B.hs
- + testsuite/tests/driver/mkdepend/T24384/lib/B.hs-boot
- testsuite/tests/driver/mkdepend/preproc.sh → testsuite/tests/driver/mkdepend/T24384/preproc.sh
- testsuite/tests/driver/mkdepend/T24384a.stdout
- + testsuite/tests/driver/mkdepend/T24384b.stdout
- + testsuite/tests/driver/mkdepend/T24384c.stdout
- testsuite/tests/driver/mkdepend/all.T
- + testsuite/tests/driver/mkdepend/format_json.sh
- + testsuite/tests/driver/mkdepend/format_makefile.sh
- + testsuite/tests/driver/mkdepend/mhu01/a/A/Base.hs
- + testsuite/tests/driver/mkdepend/mhu01/a/A/Base.hs-boot
- + testsuite/tests/driver/mkdepend/mhu01/a/A/TH.hs
- + testsuite/tests/driver/mkdepend/mhu01/a/Util.hs
- + testsuite/tests/driver/mkdepend/mhu01/b/B/Normal.hs
- + testsuite/tests/driver/mkdepend/mhu01/b/B/Splice.hs
- + testsuite/tests/driver/mkdepend/mhu01/b/Util.hs
- + testsuite/tests/driver/mkdepend/mhu01/c/C.hs
- + testsuite/tests/driver/mkdepend/mhu01/unitA
- + testsuite/tests/driver/mkdepend/mhu01/unitB
- + testsuite/tests/driver/mkdepend/mhu01/unitC
- + testsuite/tests/driver/mkdepend/mhu02/A.hs
- + testsuite/tests/driver/mkdepend/mhu02/B.hs
- + testsuite/tests/driver/mkdepend/mhu02/pp.sh
- + testsuite/tests/driver/mkdepend/mhu02/unitA
- + testsuite/tests/driver/mkdepend/mhu02/unitB
- + testsuite/tests/driver/mkdepend/mhu03/pkg/A.hs
- + testsuite/tests/driver/mkdepend/mhu03/pkg/pp.sh
- + testsuite/tests/driver/mkdepend/mhu03/unitA
- + testsuite/tests/driver/mkdepend/mkdepend01.dep-json
- + testsuite/tests/driver/mkdepend/mkdepend01.dep-makefile
- + testsuite/tests/driver/mkdepend/mkdepend01/A.hs
- + testsuite/tests/driver/mkdepend/mkdepend_mhu01.dep-json
- + testsuite/tests/driver/mkdepend/mkdepend_mhu01.dep-makefile
- + testsuite/tests/driver/mkdepend/mkdepend_mhu02.dep-json
- + testsuite/tests/driver/mkdepend/mkdepend_mhu03.dep-json
- testsuite/tests/perf/compiler/large-project/all.T
The diff was not included because it is too large.
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/1a3a4abec643ba465b06d4f85d6ff5…
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/1a3a4abec643ba465b06d4f85d6ff5…
You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help
1
0
[Git][ghc/ghc][wip/romes/async-rpc] Asynchronous RPC with external interpreter
by Rodrigo Mesquita (@alt-romes) 13 Oct '26
by Rodrigo Mesquita (@alt-romes) 13 Oct '26
13 Oct '26
Rodrigo Mesquita pushed to branch wip/romes/async-rpc at Glasgow Haskell Compiler / GHC
Commits:
9d191061 by Rodrigo Mesquita at 2026-10-13T12:32:16+01:00
Asynchronous RPC with external interpreter
There's a single pipe to connect the host compiler to the external
interpreter. On this pipe, we have historically done synchronous
request/reply communication to invoke methods on the external
interpreter.
However, with the advent of multi-threaded debugging, it has become
necessary to do asynchronous calls on the external interpreter, where we
may block on the host waiting for a reply, but the external interpreter
should handle the request asynchronously and reply when ready. For
instance, we want to start the evaluation of the debuggee but not block
the interpreter waiting for it to finish or hit a breakpoint. Or e.g. we
want to resume two threads of the debuggee simulatenously instead of
blocking waiting for the resume result of each one sequentially.
And it turns out, we were already doing some ad-hoc asynchronous
communication to evaluate Template Haskell on the external interpreter.
It used to go as an ad-hoc "send request without expecting reply",
followed by handling on the host a set of 'THMessage's sent by the
interpreter, followed by "expect a response to arrive now". It worked
because there was still a fixed order of messages, but it fits an async
model in a non-ad-hoc way.
Asynchronous communication that unblocks the external interpreter main
thread will also make it possible to resolve issues such as #18852
(exceptions leaving the external interpreter in an unrecoverrable state)
The new design
--------------
The new invariant: every single request should receive a matching reply.
The pipe is bidirectional (the ext. interpreter can receive requests,
but it can also send requests for TH evaluation; the host can send
requests, but it can also receive TH requests from the interpreter).
When a request is sent in whichever direction, it always contains a
correlation-id header. The Sender places an empty MVar on a
pending-reply map keyed by id and waits on it for a response.
See 'remoteCall' and 'remoteTHCall'.
The pipe is associated to a thread which drains the read end. Because
it's bidirectional, two things may be read off the pipe:
- A Reply, with a correlation-id header, which we match against the
pending requests map to unblock the corresponding pending-reply MVar
- A Request, which is queued to the pending-request queue.
To handle a message, we take a Request from the pending-request queue
and run a handler that produces a Reply. Currently, the handler is run
synchronously in the thread, but it is trivial to make the handler run
in a forked thread for certain ("interruptible") messages in an upcoming
patch (when it is necessary for the multi-threaded debugger).
See 'handleRemoteCall' and 'handleRemoteTHCall'.
The old interp lock now just protect the pipe writing (reads don't lock
bc they are done exclusively by drainPipe), it doesn't need nor should
lock the full remote procedure call.
- - - - -
10 changed files:
- compiler/GHC/Runtime/Interpreter.hs
- compiler/GHC/Runtime/Interpreter/JS.hs
- compiler/GHC/Runtime/Interpreter/Process.hs
- compiler/GHC/Runtime/Interpreter/Types.hs
- compiler/GHC/Runtime/Interpreter/Wasm.hs
- compiler/GHC/Tc/Gen/Splice.hs
- libraries/ghci/GHCi/Message.hs
- libraries/ghci/GHCi/Server.hs
- libraries/ghci/ghci.cabal.in
- testsuite/tests/ghci/custom-external-interpreter-commands/Main.hs
Changes:
=====================================
compiler/GHC/Runtime/Interpreter.hs
=====================================
@@ -626,11 +626,9 @@ spawnIServ conf = do
[]
(iservConfOpts conf)
interpPipe <- mkPipeFromHandles rh wh
- lock <- newMVar ()
let process = InterpProcess
{ interpHandle = ph
, interpPipe
- , interpLock = lock
}
pending_frees <- newMVar []
=====================================
compiler/GHC/Runtime/Interpreter/JS.hs
=====================================
@@ -127,11 +127,9 @@ startTHRunnerProcess interp_js settings = do
std_in <- readIORef interp_in
interpPipe <- mkPipeFromHandles rh wh
- lock <- newMVar ()
let proc = InterpProcess
{ interpHandle = hdl
, interpPipe
- , interpLock = lock
}
pure (std_in, proc)
@@ -382,7 +380,7 @@ jsLinkPlan logger tmpfs tmp_dir ar_cache link_cfg cfg inst link_plan = do
-- Haskell server.
server_started <- jsServerStarted <$> readMVar (instJSState (instExtra inst))
if server_started
- then sendMessageNoResponse inst $ LoadObj all_js
+ then sendMessage inst $ LoadObj all_js
else jsLoadFile inst all_js
----------------------------------------------------------------
=====================================
compiler/GHC/Runtime/Interpreter/Process.hs
=====================================
@@ -2,16 +2,8 @@ module GHC.Runtime.Interpreter.Process
(
-- * Message API
Message(..)
- , DelayedResponse (..)
- -- * Top-level message API (these acquire/release a lock)
, sendMessage
- , sendMessageNoResponse
- , sendMessageDelayedResponse
- , receiveDelayedResponse
- -- * Nested message API (these require the interpreter to already be locked)
- , sendAnyValue
- , receiveAnyValue
- , receiveTHMessage
+ , sendTHComputation
)
where
@@ -27,97 +19,60 @@ import GHC.Utils.Exception as Ex
import Data.Binary
import System.Exit
import System.Process
-import Control.Concurrent.MVar (MVar, withMVar, takeMVar, putMVar, isEmptyMVar)
-
-data DelayedResponse a = DelayedResponse
+import Control.Concurrent
+import Data.IORef
+import Control.Concurrent.STM
-- -----------------------------------------------------------------------------
-- Top-level Message API
--- | Send a message to the interpreter process that doesn't expect a response
--- (locks the interpreter while sending)
-sendMessageNoResponse :: ExtInterpInstance d -> Message () -> IO ()
-sendMessageNoResponse i m =
- withLock i $ writeInterpProcess (instProcess i) (putMessage m)
-
--- | Send a message to the interpreter that expects a response
--- (locks the interpreter while until the response is received)
+-- | Send a message to the interpreter and expect a response, blocking until it arrives.
+--
+-- This issues an asynchronous RPC on the external interpreter which does not
+-- block the external interpreter thread. We block here on the host because we
+-- wait for the asynchronous response to arrive.
sendMessage :: Binary a => ExtInterpInstance d -> Message a -> IO a
-sendMessage i m = withLock i $ callInterpProcess (instProcess i) m
-
--- | Send a message to the interpreter process whose response is expected later
+sendMessage i msg =
+ remoteCall (interpPipe proc) msg
+ `catchException` \(e :: SomeException) -> handleInterpProcessFailure proc e
+ where
+ proc = instProcess i
+
+-- | Send a message with a TH computation to the ext. interpreter (like
+-- 'RunTH', or 'RunModFinalizers') and handle 'THMessage' requests from the
+-- ext. interpreter to us in the meantime.
--
--- This is useful to avoid forgetting to receive the value and to ensure that
--- the type of the response isn't lost. Use receiveDelayedResponse to read it.
--- (locks the interpreter until the response is received using
--- `receiveDelayedResponse`)
-sendMessageDelayedResponse :: ExtInterpInstance d -> Message a -> IO (DelayedResponse a)
-sendMessageDelayedResponse i m = do
- lock i
- writeInterpProcess (instProcess i) (putMessage m)
- pure DelayedResponse
-
--- | Expect a delayed result to be received now
-receiveDelayedResponse :: Binary a => ExtInterpInstance d -> DelayedResponse a -> IO a
-receiveDelayedResponse i DelayedResponse = do
- ensureLocked i
- r <- readInterpProcess (instProcess i) get
- unlock i
- pure r
-
--- -----------------------------------------------------------------------------
--- Nested Message API
-
--- | Send any value (requires locked interpreter)
-sendAnyValue :: Binary a => ExtInterpInstance d -> a -> IO ()
-sendAnyValue i m = ensureLocked i >> writeInterpProcess (instProcess i) (put m)
-
--- | Expect a value to be received (requires locked interpreter)
-receiveAnyValue :: ExtInterpInstance d -> Get a -> IO a
-receiveAnyValue i get = ensureLocked i >> readInterpProcess (instProcess i) get
-
--- | Wait for a Template Haskell message (requires locked interpreter)
-receiveTHMessage :: ExtInterpInstance d -> IO THMsg
-receiveTHMessage i = ensureLocked i >> receiveAnyValue i getTHMessage
-
--- -----------------------------------------------------------------------------
-
-getLock :: ExtInterpInstance d -> MVar ()
-getLock = interpLock . instProcess
-
-withLock :: ExtInterpInstance d -> IO a -> IO a
-withLock i f = withMVar (getLock i) (const f)
-
-lock :: ExtInterpInstance d -> IO ()
-lock i = takeMVar (getLock i)
-
-unlock :: ExtInterpInstance d -> IO ()
-unlock i = putMVar (getLock i) ()
-
-ensureLocked :: ExtInterpInstance d -> IO ()
-ensureLocked i =
- isEmptyMVar (getLock i) >>= \case
- False -> panic "ensureLocked: external interpreter not locked"
- _ -> pure ()
-
-
--- | Send a 'Message' and receive the response from the interpreter process
-callInterpProcess :: Binary a => InterpProcess -> Message a -> IO a
-callInterpProcess i msg =
- remoteCall (interpPipe i) msg
- `catchException` \(e :: SomeException) -> handleInterpProcessFailure i e
-
--- | Read a value from the interpreter process
-readInterpProcess :: InterpProcess -> Get a -> IO a
-readInterpProcess i get =
- readPipe (interpPipe i) get
- `catchException` \(e :: SomeException) -> handleInterpProcessFailure i e
-
--- | Send a value to the interpreter process
-writeInterpProcess :: InterpProcess -> Put -> IO ()
-writeInterpProcess i put =
- writePipe (interpPipe i) put
- `catchException` \(e :: SomeException) -> handleInterpProcessFailure i e
+-- This function blocks until the TH computation returns a value, but spawns in
+-- the background a thread to handle the 'THMessage's until the TH computation
+-- is fully done.
+sendTHComputation
+ :: Binary a
+ => ExtInterpInstance d
+ -> Message a
+ -> (forall b. Binary b => THMessage b -> IO (Bool, b))
+ -- ^ Handle one message. Return True to end the loop.
+ -> IO a
+sendTHComputation i msg reqHandler = mask $ \restore -> do
+ -- inline "withAsync/wait"
+ var <- newEmptyTMVarIO
+ ht <- forkFinally (restore handlerLoop) (atomically . putTMVar var)
+ restore (sendMessage i msg <* atomically (readTMVar var))
+ `onException` uninterruptibleMask_ (throwTo ht CancelTHHandler)
+ where
+ proc = instProcess i
+ handleOne doneRef = do
+ handleRemoteTHCall (interpPipe proc) $ \req -> do
+ (stop, res) <- reqHandler req
+ writeIORef doneRef stop
+ return res
+ handlerLoop = do
+ doneRef <- newIORef False
+ let loop = do
+ handleOne doneRef
+ isDone <- readIORef doneRef
+ if isDone then return ()
+ else loop
+ in loop
handleInterpProcessFailure :: InterpProcess -> SomeException -> IO a
handleInterpProcessFailure i e = do
@@ -130,3 +85,8 @@ handleInterpProcessFailure i e = do
terminateProcess hdl
_ <- waitForProcess hdl
throw e
+
+data CancelTHHandler = CancelTHHandler deriving Show
+instance Exception CancelTHHandler where
+ fromException = asyncExceptionFromException
+ toException = asyncExceptionToException
=====================================
compiler/GHC/Runtime/Interpreter/Types.hs
=====================================
@@ -111,7 +111,6 @@ type WasmInterp = ExtInterpState WasmInterpConfig ()
data InterpProcess = InterpProcess
{ interpPipe :: !Pipe -- ^ Pipe to communicate with the server
, interpHandle :: !ProcessHandle -- ^ Process handle of the server
- , interpLock :: !(MVar ()) -- ^ Lock to prevent concurrent access to the stream
}
-- | Status of an external interpreter
=====================================
compiler/GHC/Runtime/Interpreter/Wasm.hs
=====================================
@@ -73,14 +73,12 @@ spawnWasmInterp WasmInterpConfig {..} = do
hSetBuffering rh NoBuffering
interpPipe <- mkPipeFromHandles rh wh
pending_frees <- newMVar []
- lock <- newMVar ()
pure
$ ExtInterpInstance
{ instProcess =
InterpProcess
{ interpHandle = ph,
- interpPipe,
- interpLock = lock
+ interpPipe
},
instPendingFrees = pending_frees,
instExtra = ()
=====================================
compiler/GHC/Tc/Gen/Splice.hs
=====================================
@@ -170,6 +170,7 @@ import Data.IORef
import GHC.Parser.HaddockLex (lexHsDoc)
import GHC.Parser (parseIdentifier)
import GHC.Rename.Doc (rnHsDoc)
+import Data.Function
{-
Note [Template Haskell state diagram]
@@ -1165,11 +1166,12 @@ runRemoteModFinalizers (ThModFinalizers finRefs) = do
case th_state of
Nothing -> return () -- TH was not started, nothing to do
Just fhv -> do
- r <- liftIO $ withForeignRef fhv $ \st ->
- withForeignRefs finRefs $ \qrefs ->
- sendMessageDelayedResponse inst (RunModFinalizers st qrefs)
- () <- runRemoteTH inst []
- qr <- liftIO $ receiveDelayedResponse inst r
+ qr <- withRunInIO $ \runInIO -> do
+ recovers <- newIORef []
+ withForeignRef fhv $ \st ->
+ withForeignRefs finRefs $ \qrefs ->
+ sendTHComputation inst (RunModFinalizers st qrefs) $ \msg ->
+ runInIO (runRemoteTH recovers msg)
checkQResult qr
runQResult
@@ -1824,48 +1826,47 @@ runTH ty fhv = do
rstate <- getTHState inst
loc <- location
-- run a remote TH request
- r <- liftIO $
+ qr <- withRunInIO $ \runInIO -> do
+ recovers <- newIORef []
withForeignRef rstate $ \state_hv ->
- withForeignRef fhv $ \q_hv ->
- sendMessageDelayedResponse inst (RunTH state_hv q_hv ty (Just loc))
- -- respond to requests from the interpreter
- runRemoteTH inst []
- -- get the final result
- qr <- liftIO $ receiveDelayedResponse inst r
+ withForeignRef fhv $ \q_hv ->
+ sendTHComputation inst (RunTH state_hv q_hv ty (Just loc)) $ \msg ->
+ runInIO (runRemoteTH recovers msg)
bs <- checkQResult qr
return $! runGet get (LB.fromStrict bs)
--- | communicate with a remotely-running TH computation until it finishes.
+-- | Communicate with a remotely-running TH computation until it finishes.
-- See Note [Remote Template Haskell] in libraries/ghci/GHCi/TH.hs.
runRemoteTH
- :: ExtInterpInstance d
- -> [Messages TcRnMessage] -- saved from nested calls to qRecover
- -> TcM ()
-runRemoteTH inst recovers = do
- THMsg msg <- liftIO $ receiveTHMessage inst
+ :: Binary a
+ => IORef [Messages TcRnMessage] -- ^ saved from nested calls to qRecover
+ -> THMessage a
+ -> TcM (Bool, a)
+runRemoteTH recovers msg = do
case msg of
- RunTHDone -> return ()
+ RunTHDone -> return (True, ())
StartRecover -> do -- Note [TH recover with -fexternal-interpreter]
v <- getErrsVar
msgs <- readTcRef v
writeTcRef v emptyMessages
- runRemoteTH inst (msgs : recovers)
+ modifyIORef recovers (msgs:) & liftIO
+ return (False, ())
EndRecover caught_error -> do
- let (prev_msgs, rest) = case recovers of
- [] -> panic "EndRecover"
- a : b -> (a,b)
+ (prev_msgs, rest) <- liftIO $
+ readIORef recovers >>= \case
+ [] -> panic "EndRecover"
+ a : b -> pure (a,b)
v <- getErrsVar
warn_msgs <- getWarningMessages <$> readTcRef v
-- keep the warnings only if there were no errors
writeTcRef v $ if caught_error
then prev_msgs
else mkMessages warn_msgs `unionMessages` prev_msgs
- runRemoteTH inst rest
+ writeIORef recovers rest & liftIO
+ return (False, ())
_other -> do
- r <- handleTHMessage msg
- liftIO $ sendAnyValue inst r
- runRemoteTH inst recovers
+ (False,) <$> handleTHMessage msg
-- | Check a QResult
checkQResult :: QResult a -> TcM a
=====================================
libraries/ghci/GHCi/Message.hs
=====================================
@@ -1,6 +1,7 @@
-{-# LANGUAGE GADTs, DeriveGeneric, StandaloneDeriving, ScopedTypeVariables,
- GeneralizedNewtypeDeriving, ExistentialQuantification, RecordWildCards,
- CPP, NamedFieldPuns #-}
+{-# LANGUAGE RecursiveDo, GADTs, DeriveGeneric, StandaloneDeriving, ScopedTypeVariables,
+ GeneralizedNewtypeDeriving, ExistentialQuantification, RecordWildCards, CPP,
+ NamedFieldPuns, LambdaCase, TypeAbstractions, TypeApplications,
+ LinearTypes, RankNTypes, UnboxedTuples, BlockArguments, TupleSections, MagicHash #-}
{-# OPTIONS_GHC -fno-warn-name-shadowing -fno-warn-orphans #-}
-- |
@@ -22,10 +23,25 @@ module GHCi.Message
, ResumeContext(..)
, QState(..)
, getMessage, putMessage, getTHMessage, putTHMessage
- , Pipe, mkPipeFromHandles, mkPipeFromContinuations, remoteCall, remoteTHCall, readPipe, writePipe
+
+ , Pipe(..), mkPipeFromHandles, mkPipeFromContinuations, closePipe
+ , remoteCall, remoteTHCall
+ , handleRemoteCall, handleRemoteTHCall
+ , RemoteProcessException(..)
+
, BreakModule
, BreakUnitId
, LoadedDLL
+
+ -- * Low-level interface for async bidirectional RPC
+ , remoteAsyncReq
+ , handleAsyncReq
+ -- ** Consume `reply` exactly once
+ , WriteReply -- abstract
+ , writeReply
+ -- ** Async matching internals
+ , AsyncManager(..), CorrelationId
+ , IddMsg(..), newAsyncManager
) where
import Prelude -- See note [Why do we import Prelude here?]
@@ -45,6 +61,7 @@ import qualified GHC.Exts.Heap as Heap
import GHC.ForeignSrcLang
import GHC.Fingerprint
import GHC.Conc (pseq, par)
+import GHC.IO (IO(..))
import Control.Concurrent
import Control.DeepSeq
import Control.Exception
@@ -63,6 +80,8 @@ import Data.Dynamic
import Data.Typeable (TypeRep)
import Data.IORef
import Data.Map (Map)
+import Data.IntMap (IntMap)
+import qualified Data.IntMap as IntMap
import Foreign
import GHC.Generics
import GHC.Stack.CCS
@@ -71,6 +90,11 @@ import qualified GHC.Boot.TH.Monad as TH
import System.Exit
import System.IO
import System.IO.Error
+import Control.Monad
+import Control.Concurrent.STM
+import Data.Bifunctor
+import GHC.Base (State#)
+import GHC.Exts (RealWorld)
-- -----------------------------------------------------------------------------
-- The RPC protocol between GHC and the interactive server
@@ -693,42 +717,252 @@ serializeBCOs rbcos = parMap doChunk (chunkList 100 rbcos)
-- | An opaque pipe for bidirectional binary data transmission.
data Pipe = Pipe
- { getSome :: !(IO ByteString)
- , putAll :: !(B.Builder -> IO ())
- , pipeLeftovers :: !(IORef (Maybe ByteString))
+ { getSome :: !(IO ByteString)
+ , putAll :: !(B.Builder -> IO ())
+ , pipeLeftovers :: !(IORef (Maybe ByteString))
+ , pipeLock :: !(MVar ()) -- ^ Lock to prevent concurrent writes to the stream
+ , pipeAsyncMngr :: !AsyncManager
+ , pipeCleanupRW :: !(IO ())
+ -- ^ Clean up the read/write resources, e.g. closes the handles if 'mkPipeFromHandles'
}
-- | Make a 'Pipe' from a 'Handle' to read and a 'Handle' to write.
+-- Must call 'closePipe' before closing the handles, otherwise 'hClose' will
+-- deadlock with the thread blocked draining the read end.
mkPipeFromHandles :: Handle -> Handle -> IO Pipe
mkPipeFromHandles pipeRead pipeWrite = do
let getSome = B.hGetSome pipeRead (32*1024)
putAll b = do
B.hPutBuilder pipeWrite b
hFlush pipeWrite
- pipeLeftovers <- newIORef Nothing
- pure $ Pipe { getSome, putAll, pipeLeftovers }
+ mkPipeFromContinuations getSome putAll (hClose pipeRead >> hClose pipeWrite)
+
+-- | Make a 'Pipe' from a reader function and a writer function, plus an action
+-- to clean up the reader and writer resources. The pipe should be closed with
+-- 'closePipe', which will call this clean up action too.
+mkPipeFromContinuations :: IO ByteString -> (B.Builder -> IO ()) -> IO () -> IO Pipe
+mkPipeFromContinuations getSome putAll pipeCleanupRW = mdo
+ pipeLeftovers <- newIORef Nothing
+ pipeLock <- newMVar ()
+ mdo
+ pipeAsyncMngr <- newAsyncManager p
+ let p = Pipe { .. }
+ pure p
+
+-- | Cleanly close this pipe, terminating the async manager's drain-read-end thread.
+-- This doesn't close the handles
+closePipe :: Pipe -> IO ()
+closePipe Pipe{..} = do
+ killThread (asyncDrainTid pipeAsyncMngr) -- releases handle lock taken by reading
+ pipeCleanupRW -- hClose won't deadlock now that the drainThread is killed
+
+-- | Send a 'Message' request to and wait for a reply from the interpreter
+-- (blocks on the calling side, doesn't necessarily block the interpreter)
+remoteCall :: Binary a => Pipe -> Message a -> IO a
+remoteCall pipe msg = either throwIO (pure . runGet get)
+ =<< remoteAsyncReq pipe (putMessage msg)
+
+-- | Handle a single 'Message' request on the interpreter and reply to the host
+handleRemoteCall :: Pipe
+ -> (Msg -> IO Msg) {-^ Msg hook -}
+ -> (forall a. (Show a, Binary a) => Message a -> IO a)
+ -> IO ()
+handleRemoteCall pipe hook k =
+ handleAsyncReq pipe getMessage \writer msg0 -> IO \s0 ->
+ case hook msg0 |#> s0 of -- unIO for linearity checking
+ (# s1, Msg msg #) -> case k msg |#> s1 of
+ (# s2, a #) -> writeReply writer (put a) |#> s2
+
+-- | Send a 'THMessage' request to the host and wait for a reply from the host
+-- to the interpreter
+remoteTHCall :: Binary a => Pipe -> THMessage a -> IO a
+remoteTHCall pipe msg = either throwIO (pure . runGet get)
+ =<< remoteAsyncReq pipe (putTHMessage msg)
+
+-- | Handle a 'THMessage' request on the host and reply to the interpreter.
+handleRemoteTHCall
+ :: Pipe -> (forall a. (Show a, Binary a) => THMessage a -> IO a) -> IO ()
+handleRemoteTHCall pipe k =
+ handleAsyncReq pipe getTHMessage \writer (THMsg msg) -> IO \s0 ->
+ case k msg |#> s0 of -- unIO for linearity checking
+ (# s1, a #) -> writeReply writer (put a) |#> s1
+
+--------------------------------------------------------------------------------
+-- * Matching requests with async replies
+--------------------------------------------------------------------------------
+
+-- | Send an asynchronous request and block waiting for a matching reply
+remoteAsyncReq :: Pipe -> Put -> IO (Either SomeException LB.ByteString)
+remoteAsyncReq pipe msg = do
+
+ let pendingReplies = pendingReply (pipeAsyncMngr pipe)
+
+ -- Mark pending request
+ uq <- freshReqId (pipeAsyncMngr pipe)
+ wv <- newEmptyMVar
+ atomicModifyIORef' pendingReplies $
+ \m -> (IntMap.insert uq wv m, ())
+
+ -- Write request
+ writePipe pipe (Request uq (runPut msg))
+ `onException`
+ -- the req wasn't done, or no one will be reading the reply:
+ -- in either case, drop it from the pending map.
+ atomicModifyIORef' pendingReplies
+ (\m -> (IntMap.delete uq m, ()))
+
+ -- Block waiting for reply
+ takeMVar wv
+
+-- | Receive a single asynchronous request and send a matching reply
+-- asynchronously (the handler may fork a computation that eventually replies
+-- and can return before actually replying, as long as it eventually does).
+--
+-- The handler receives as an argument the function to use to reply to the request.
+-- It must be used exactly once (all requests expect a reply, and you can't
+-- reply more than once). Linear types enforce the handler replies in every
+-- branch, and if the handler otherwise throws, it is handled by 'withAsyncHandler'.
+handleAsyncReq :: Pipe -> Get a -> (WriteReply Put %1 -> a -> IO r) -> IO r
+handleAsyncReq pipe getit k = do
+ ack <- newEmptyMVar
+
+ -- Block waiting for request
+ (uq, bs) <- atomically (readTQueue (requestsQueue (pipeAsyncMngr pipe)))
+
+ let reply p = do
+ writePipe pipe (Reply uq (bimap toSerializableException runPut p))
+ putMVar ack ()
+
+ -- Run the continuation which must reply exactly once with 'writeReply'
+ case runGetOrFail getit bs of
+ Left (_, _, err) -> do
+ let exc = DecodeAsyncReqException err
+ reply (Left (toException exc))
+ throwIO exc
+ Right (_, _, it) ->
+ k (WriteReply reply) it
+
+ -- If there's an exception before the reply is sent, reply with an error instead.
+ `catch` \(e :: SomeException) -> do
+
+ tryTakeMVar ack >>= \case
+ Just () ->
+ -- Already replied
+ throwIO e
+ Nothing -> do
+ -- Reply with an error
+ reply (Left e)
+ throwIO e
+
+-- | An exception that can be thrown on the host process representing an
+-- expection that happened on the remote one.
+newtype RemoteProcessException = RemoteProcessException SerializableException deriving Show
+instance Exception RemoteProcessException
+
+-- | 'handleSyncReq' failed to decode an incoming request payload using the given @'Get' a@
+newtype DecodeAsyncReqException = DecodeAsyncReqException String deriving Show
+instance Exception DecodeAsyncReqException
+
+-- ** Async matching "internals" -----------------------------------------------
+
+data AsyncManager = AsyncManager
+ { pendingReply :: !(IORef (IntMap (MVar (Either SomeException LB.ByteString))))
+ -- ^ When a request is sent, keep track that we're expecting a reply. The
+ -- MVar is signaled when the reply arrives. The exception can originate
+ -- from the host process, or from the remote one (with 'RemoteProcessException').
+ , requestsQueue :: !(TQueue (CorrelationId, LB.ByteString))
+ -- ^ A queue of pending received requests. When a request is received,
+ -- queue it here. A request handler will pop the next request from it.
+ -- (Nothing except the 'drainPipe' thread should read from the pipe).
+ , asyncDrainTid :: !ThreadId
+ , asyncNextReq :: !(IORef Int)
+ }
--- | Make a 'Pipe' from a reader function and a writer function.
-mkPipeFromContinuations :: IO ByteString -> (B.Builder -> IO ()) -> IO Pipe
-mkPipeFromContinuations getSome putAll = do
- pipeLeftovers <- newIORef Nothing
- pure $ Pipe { getSome, putAll, pipeLeftovers }
+type CorrelationId = Int
-remoteCall :: Binary a => Pipe -> Message a -> IO a
-remoteCall pipe msg = do
- writePipe pipe (putMessage msg)
- readPipe pipe get
+-- | We have a single type for requests and replies because both the host and
+-- interpreter may send requests *and* replies on the write end *and* on recv
+-- both on the read end.
+--
+-- For instance:
+-- 1. The host sends a RunTH request on its Host Write End (HWE)
+-- 2. The interpreter sends a NewName TH request on its Interpreter Write End (IWE)
+-- 3. The host sends a reply to the TH request on its HWE
+-- 4. The interpreter sends the reply to the RunTH request on its IWE
+data IddMsg = Request !CorrelationId LB.ByteString
+ | Reply !CorrelationId (Either SerializableException LB.ByteString)
+
+instance Binary IddMsg where
+ put (Request i bs) = putWord8 0 >> put i >> put bs
+ put (Reply i bs) = putWord8 1 >> put i >> put bs
+ get = do
+ b <- getWord8
+ case b of
+ 0 -> Request <$> get <*> get
+ 1 -> Reply <$> get <*> get
+ _ -> error $ "impossible: Binary IddMsg - " ++ show b
-writePipe :: Pipe -> Put -> IO ()
-writePipe Pipe{..} put = putAll $ execPut put
+newAsyncManager :: Pipe -> IO AsyncManager
+newAsyncManager pipe = do
+ asyncNextReq <- newIORef 0
+ pendingReply <- newIORef IntMap.empty
+ requestsQueue <- newTQueueIO
-remoteTHCall :: Binary a => Pipe -> THMessage a -> IO a
-remoteTHCall pipe msg = do
- writePipe pipe (putTHMessage msg)
- readPipe pipe get
+ mdo
+ -- Fork thread to drain read end.
+ -- (cleaned up on 'closePipe')
+ asyncDrainTid <- forkIO (drainPipe pipe am)
+
+ let am = AsyncManager{..}
+ pure am
-readPipe :: Pipe -> Get a -> IO a
-readPipe Pipe{..} get = do
+-- | Drains the read end of the Pipe and match 'Reply'es against pending 'Request's.
+--
+-- In pratice, we'll have both a thread draining the read end of the host's
+-- 'Pipe' and a thread draining the read end of the interpreter's 'Pipe', since
+-- both processes construct a 'Pipe' and send and receive messages:
+drainPipe :: Pipe -> AsyncManager -> IO ()
+drainPipe p am = flip catch cancelPending $ forever $ do
+ readPipe p >>= \case
+ -- Receive a reply to a pending notification
+ Reply uq bs -> do
+ mwv <- atomicModifyIORef' (pendingReply am) $ \m -> do
+ case IntMap.lookup uq m of
+ Nothing -> (m, Nothing)
+ Just wv -> (IntMap.delete uq m, Just wv)
+ case mwv of
+ Just wv ->
+ putMVar wv (bimap (toException . RemoteProcessException) id bs) -- signal matching request
+ Nothing ->
+ -- the req was dropped from pending bc there was an exception, and no
+ -- one's listening any more. the reply is ignored
+ pure ()
+ -- Receive a request and queue it
+ Request uq bs ->
+ atomically $ writeTQueue (requestsQueue am) (uq,bs)
+ where
+ cancelPending (e::SomeException) = do
+ pendings <- readIORef (pendingReply am)
+ mapM_ (\p -> putMVar p (Left e {-host exception-})) pendings
+
+freshReqId :: AsyncManager -> IO CorrelationId
+freshReqId AsyncManager{asyncNextReq} =
+ atomicModifyIORef' asyncNextReq $ \i -> (i+1, i)
+
+--------------------------------------------------------------------------------
+-- * Pipe thread-safe read/write
+--------------------------------------------------------------------------------
+-- All messages on the wire are 'IddMsg', which contain a correlation header
+-- and identify whether they are requests or replies.
+
+writePipe :: Pipe -> IddMsg -> IO ()
+writePipe p@Pipe{..} = withLock p . putAll . execPut . put
+
+-- | Only the 'drainPipe' thread should read from the pipe directly. Other
+-- functions should wait for the matching reply to be delivered asynchronously.
+-- (see 'withAsyncReq'). It doesn't lock to read bc it assumes single reader is 'drainPipe'.
+readPipe :: Pipe -> IO IddMsg
+readPipe Pipe{..} = do
leftovers <- readIORef pipeLeftovers
m <- getBin getSome get leftovers
case m of
@@ -758,3 +992,22 @@ getBin getsome get leftover = go leftover (runGetIncremental get)
else go Nothing (fun (Just b))
go _lft (Fail _rest _off str) =
throwIO (ErrorCall ("getBin: " ++ str))
+
+withLock :: Pipe -> IO c -> IO c
+withLock Pipe{..} = bracket (takeMVar pipeLock) (putMVar pipeLock) . const
+
+--------------------------------------------------------------------------------
+-- * Consume `reply` exactly once
+--------------------------------------------------------------------------------
+
+-- | Every request should get exactly one reply.
+data WriteReply a where
+ WriteReply :: (Either SomeException a -> IO ()) %1 -> WriteReply a
+
+-- | Write a reply exactly once (consuming the ability to write more replies)
+writeReply :: WriteReply a %1 -> a -> IO ()
+writeReply (WriteReply r) = \x -> r (Right x)
+
+-- | Infix, linear, 'unIO'
+(|#>) :: IO a %1 -> State# RealWorld -> (# State# RealWorld, a #)
+(|#>) (IO fs) s = fs s
=====================================
libraries/ghci/GHCi/Server.hs
=====================================
@@ -1,4 +1,4 @@
-{-# LANGUAGE CPP, RankNTypes, RecordWildCards, GADTs, ScopedTypeVariables #-}
+{-# LANGUAGE LambdaCase, CPP, RankNTypes, RecordWildCards, GADTs, ScopedTypeVariables #-}
module GHCi.Server
( MessageHook
, CustomMessageHandler
@@ -15,6 +15,7 @@ import GHCi.Signals
import GHCi.TH
import GHCi.Message
import Data.ByteString (ByteString)
+import Data.IORef
#if defined(wasm32_HOST_ARCH)
import qualified Data.ByteString.Builder as B
import qualified Data.ByteString.Internal as B
@@ -60,27 +61,26 @@ servWithCustom
-> (forall a .IO a -> IO a)
-> CustomMessageHandler
-> IO ()
-servWithCustom verbose hook pipe restore customHandler = loop
+servWithCustom verbose hook pipe restore customHandler = newIORef False >>= loop
where
- loop = do
+ loop doneRef = do
when verbose $ trace "reading pipe..."
- Msg msg <- readPipe pipe getMessage >>= hook
+ handleRemoteCall pipe hook $ \(msg :: Message a) -> do
- discardCtrlC
+ discardCtrlC
- when verbose $ trace ("msg: " ++ (show msg))
- case msg of
- CustomMessage tag payload -> handleCustom tag payload
- Shutdown -> return ()
- RunTH st q ty loc -> wrapRunTH $ runTH pipe st q ty loc
- RunModFinalizers st qrefs -> wrapRunTH $ runModFinalizerRefs pipe st qrefs
- _other -> run msg >>= reply
+ when verbose $ trace ("msg: " ++ (show msg))
+ case msg of
+ CustomMessage tag payload -> handleCustom tag payload
+ Shutdown -> writeIORef doneRef True >> reply ()
+ -- TODO: These two should probably be done off the main thread
+ RunTH st q ty loc -> wrapRunTH $ runTH pipe st q ty loc
+ RunModFinalizers st qrefs -> wrapRunTH $ runModFinalizerRefs pipe st qrefs
+ _other -> run msg >>= reply
- reply :: forall a. (Binary a, Show a) => a -> IO ()
- reply r = do
- when verbose $ trace ("writing pipe: " ++ show r)
- writePipe pipe (put r)
- loop
+ isDone <- readIORef doneRef
+ if isDone then return ()
+ else loop doneRef
handleCustom tag payload = do
mresp <- customHandler tag payload
@@ -89,17 +89,22 @@ servWithCustom verbose hook pipe restore customHandler = loop
Nothing ->
error $ "GHCi.Server: unhandled CustomMessage with tag " ++ show tag
+ reply :: forall a. Show a => a -> IO a
+ reply r = do
+ when verbose $ trace ("writing pipe: " ++ show r)
+ return r -- the result of the handler gets sent
+
-- Run some TH code, which may interact with GHC by sending
-- THMessage requests, and then finally send RunTHDone followed by a
-- QResult. For an overview of how TH works with Remote GHCi, see
-- Note [Remote Template Haskell] in libraries/ghci/GHCi/TH.hs.
- wrapRunTH :: forall a. (Binary a, Show a) => IO a -> IO ()
+ wrapRunTH :: forall a. (Binary a, Show a) => IO a -> IO (QResult a)
wrapRunTH io = do
when verbose $ trace "wrapRunTH..."
r <- try io
when verbose $ trace "wrapRunTH done."
when verbose $ trace "writing RunTHDone."
- writePipe pipe (putTHMessage RunTHDone)
+ remoteTHCall pipe RunTHDone
case r of
Left e
| Just (GHCiQException _ err) <- fromException e -> do
@@ -179,7 +184,7 @@ defaultServerWithCustom customHandler = do
#if defined(wasm32_HOST_ARCH)
-- See Note [wasm ghci signal handlers] for details
installSignalHandlers $ js_register_signal_handler cb_sig
- pipe <- mkPipeFromContinuations (recv_buf cb_recv) (send_buf cb_send)
+ pipe <- mkPipeFromContinuations (recv_buf cb_recv) (send_buf cb_send) (pure ())
#else
when verbose $
printf "GHC iserv starting (in: %s; out: %s)\n" (show inh) (show outh)
=====================================
libraries/ghci/ghci.cabal.in
=====================================
@@ -95,7 +95,8 @@ library
filepath >= 1.4 && < 1.6,
ghc-boot == @ProjectVersionMunged@,
ghc-heap >= 9.10.1 && <=@ProjectVersionMunged@,
- transformers >= 0.5 && < 0.7
+ transformers >= 0.5 && < 0.7,
+ stm >= 2.4 && < 2.6
if flag(bootstrap)
build-depends:
=====================================
testsuite/tests/ghci/custom-external-interpreter-commands/Main.hs
=====================================
@@ -12,6 +12,7 @@ import GHCi.Message
, mkPipeFromHandles
, remoteCall
, Pipe
+ , closePipe
)
import GHCi.Server
( CustomMessageHandler
@@ -153,14 +154,13 @@ customMessage pipe c = do
runClient :: Int -> IO ()
runClient input = do
serverExe <- getExecutablePath
- withServer serverExe $ \hFromServer hToServer -> do
- pipe <- mkPipeFromHandles hFromServer hToServer
+ withServer serverExe $ \pipe -> do
res <- customMessage pipe (SquareCommand input)
putStrLn $ "Square returned: " ++ show res
res2 <- customMessage pipe (MulCommand 2 res)
putStrLn $ "Mul returned: " ++ show res2
-withServer :: FilePath -> (Handle -> Handle -> IO a) -> IO a
+withServer :: FilePath -> (Pipe -> IO a) -> IO a
withServer serverExe action = do
(ghcRead, serverWrite) <- createPipe
(serverRead, ghcWrite) <- createPipe
@@ -171,13 +171,12 @@ withServer serverExe action = do
, std_out = Inherit
, std_err = Inherit
}
- bracket (mkHandles ghcRead ghcWrite)
- (\(hFromServer, hToServer) -> do
- hClose hFromServer
- hClose hToServer
+ bracket (mkHandles ghcRead ghcWrite >>= uncurry mkPipeFromHandles)
+ (\pipe -> do
+ closePipe pipe
terminateProcess ph
void (waitForProcess ph))
- (\(hFromServer, hToServer) -> action hFromServer hToServer)
+ (\pipe -> action pipe)
where
mkHandles r w = do
hR <- fdToHandle r
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/9d19106179bdf00b60e163603e5a51d…
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/9d19106179bdf00b60e163603e5a51d…
You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help
1
0
[Git][ghc/ghc][wip/fendor/dep-json] 5 commits: Add json output format
by Hannes Siebenhandl (@fendor) 13 Oct '26
by Hannes Siebenhandl (@fendor) 13 Oct '26
13 Oct '26
Hannes Siebenhandl pushed to branch wip/fendor/dep-json at Glasgow Haskell Compiler / GHC
Commits:
b087f117 by fendor at 2026-10-13T13:13:39+02:00
Add json output format
- - - - -
08abb892 by fendor at 2026-10-13T13:13:39+02:00
Add support for multiple home units
- - - - -
2f5001f2 by fendor at 2026-10-13T13:13:39+02:00
Record OPTIONS_GHC in the `PreprocessingNode`
This allows user to treat modules if they use -XTemplateHaskell, for
example.
- - - - -
3589630b by fendor at 2026-10-13T13:13:39+02:00
Deprecate doMkDependHS
- - - - -
1a3a4abe by fendor at 2026-10-13T13:13:39+02:00
Testsuite changes
- - - - -
63 changed files:
- + changelog.d/make-depends-json
- compiler/GHC/Driver/DynFlags.hs
- compiler/GHC/Driver/MakeFile.hs
- compiler/GHC/Driver/Pipeline/Execute.hs
- compiler/GHC/Driver/Pipeline/Phases.hs
- compiler/GHC/Driver/Session.hs
- compiler/GHC/Driver/Session/Units.hs
- compiler/GHC/Utils/Json.hs
- + docs/users_guide/make-depends-json-schema-1_0.json
- docs/users_guide/separate_compilation.rst
- ghc/Main.hs
- testsuite/driver/testglobals.py
- testsuite/driver/testlib.py
- testsuite/tests/driver/mkdepend/Makefile
- + testsuite/tests/driver/mkdepend/README.md
- testsuite/tests/driver/mkdepend/A.hs → testsuite/tests/driver/mkdepend/T24384/A.hs
- testsuite/tests/driver/mkdepend/C.hs → testsuite/tests/driver/mkdepend/T24384/C.hs
- testsuite/tests/driver/mkdepend/C.hs-boot → testsuite/tests/driver/mkdepend/T24384/C.hs-boot
- testsuite/tests/driver/mkdepend/D.hs → testsuite/tests/driver/mkdepend/T24384/D.hs
- testsuite/tests/driver/mkdepend/E.hs → testsuite/tests/driver/mkdepend/T24384/E.hs
- testsuite/tests/driver/mkdepend/dep-internal/DepInternal.hs → testsuite/tests/driver/mkdepend/T24384/dep-internal/DepInternal.hs
- testsuite/tests/driver/mkdepend/dep-internal/dep-internal.pkg → testsuite/tests/driver/mkdepend/T24384/dep-internal/dep-internal.pkg
- testsuite/tests/driver/mkdepend/dep-public/DepPublic.hs → testsuite/tests/driver/mkdepend/T24384/dep-public/DepPublic.hs
- testsuite/tests/driver/mkdepend/dep-public/dep-public.pkg → testsuite/tests/driver/mkdepend/T24384/dep-public/dep-public.pkg
- testsuite/tests/driver/mkdepend/dep/Dep.hs → testsuite/tests/driver/mkdepend/T24384/dep/Dep.hs
- testsuite/tests/driver/mkdepend/dep/dep.pkg → testsuite/tests/driver/mkdepend/T24384/dep/dep.pkg
- testsuite/tests/driver/mkdepend/foo.h → testsuite/tests/driver/mkdepend/T24384/foo.h
- testsuite/tests/driver/mkdepend/lib/B.hs → testsuite/tests/driver/mkdepend/T24384/lib/B.hs
- + testsuite/tests/driver/mkdepend/T24384/lib/B.hs-boot
- testsuite/tests/driver/mkdepend/preproc.sh → testsuite/tests/driver/mkdepend/T24384/preproc.sh
- testsuite/tests/driver/mkdepend/T24384a.stdout
- + testsuite/tests/driver/mkdepend/T24384b.stdout
- + testsuite/tests/driver/mkdepend/T24384c.stdout
- testsuite/tests/driver/mkdepend/all.T
- + testsuite/tests/driver/mkdepend/format_json.sh
- + testsuite/tests/driver/mkdepend/format_makefile.sh
- + testsuite/tests/driver/mkdepend/mhu01/a/A/Base.hs
- + testsuite/tests/driver/mkdepend/mhu01/a/A/Base.hs-boot
- + testsuite/tests/driver/mkdepend/mhu01/a/A/TH.hs
- + testsuite/tests/driver/mkdepend/mhu01/a/Util.hs
- + testsuite/tests/driver/mkdepend/mhu01/b/B/Normal.hs
- + testsuite/tests/driver/mkdepend/mhu01/b/B/Splice.hs
- + testsuite/tests/driver/mkdepend/mhu01/b/Util.hs
- + testsuite/tests/driver/mkdepend/mhu01/c/C.hs
- + testsuite/tests/driver/mkdepend/mhu01/unitA
- + testsuite/tests/driver/mkdepend/mhu01/unitB
- + testsuite/tests/driver/mkdepend/mhu01/unitC
- + testsuite/tests/driver/mkdepend/mhu02/A.hs
- + testsuite/tests/driver/mkdepend/mhu02/B.hs
- + testsuite/tests/driver/mkdepend/mhu02/pp.sh
- + testsuite/tests/driver/mkdepend/mhu02/unitA
- + testsuite/tests/driver/mkdepend/mhu02/unitB
- + testsuite/tests/driver/mkdepend/mhu03/pkg/A.hs
- + testsuite/tests/driver/mkdepend/mhu03/pkg/pp.sh
- + testsuite/tests/driver/mkdepend/mhu03/unitA
- + testsuite/tests/driver/mkdepend/mkdepend01.dep-json
- + testsuite/tests/driver/mkdepend/mkdepend01.dep-makefile
- + testsuite/tests/driver/mkdepend/mkdepend01/A.hs
- + testsuite/tests/driver/mkdepend/mkdepend_mhu01.dep-json
- + testsuite/tests/driver/mkdepend/mkdepend_mhu01.dep-makefile
- + testsuite/tests/driver/mkdepend/mkdepend_mhu02.dep-json
- + testsuite/tests/driver/mkdepend/mkdepend_mhu03.dep-json
- testsuite/tests/perf/compiler/large-project/all.T
The diff was not included because it is too large.
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/da92068af7157a0e0f9290d9ae7775…
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/da92068af7157a0e0f9290d9ae7775…
You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help
1
0
[Git][ghc/ghc][wip/romes/async-rpc] Asynchronous RPC with external interpreter
by Rodrigo Mesquita (@alt-romes) 13 Oct '26
by Rodrigo Mesquita (@alt-romes) 13 Oct '26
13 Oct '26
Rodrigo Mesquita pushed to branch wip/romes/async-rpc at Glasgow Haskell Compiler / GHC
Commits:
daab9334 by Rodrigo Mesquita at 2026-10-13T12:06:41+01:00
Asynchronous RPC with external interpreter
There's a single pipe to connect the host compiler to the external
interpreter. On this pipe, we have historically done synchronous
request/reply communication to invoke methods on the external
interpreter.
However, with the advent of multi-threaded debugging, it has become
necessary to do asynchronous calls on the external interpreter, where we
may block on the host waiting for a reply, but the external interpreter
should handle the request asynchronously and reply when ready. For
instance, we want to start the evaluation of the debuggee but not block
the interpreter waiting for it to finish or hit a breakpoint. Or e.g. we
want to resume two threads of the debuggee simulatenously instead of
blocking waiting for the resume result of each one sequentially.
And it turns out, we were already doing some ad-hoc asynchronous
communication to evaluate Template Haskell on the external interpreter.
It used to go as an ad-hoc "send request without expecting reply",
followed by handling on the host a set of 'THMessage's sent by the
interpreter, followed by "expect a response to arrive now". It worked
because there was still a fixed order of messages, but it fits an async
model in a non-ad-hoc way.
Asynchronous communication that unblocks the external interpreter main
thread will also make it possible to resolve issues such as #18852
(exceptions leaving the external interpreter in an unrecoverrable state)
The new design
--------------
The new invariant: every single request should receive a matching reply.
The pipe is bidirectional (the ext. interpreter can receive requests,
but it can also send requests for TH evaluation; the host can send
requests, but it can also receive TH requests from the interpreter).
When a request is sent in whichever direction, it always contains a
correlation-id header. The Sender places an empty MVar on a
pending-reply map keyed by id and waits on it for a response.
See 'remoteCall' and 'remoteTHCall'.
The pipe is associated to a thread which drains the read end. Because
it's bidirectional, two things may be read off the pipe:
- A Reply, with a correlation-id header, which we match against the
pending requests map to unblock the corresponding pending-reply MVar
- A Request, which is queued to the pending-request queue.
To handle a message, we take a Request from the pending-request queue
and run a handler that produces a Reply. Currently, the handler is run
synchronously in the thread, but it is trivial to make the handler run
in a forked thread for certain ("interruptible") messages in an upcoming
patch (when it is necessary for the multi-threaded debugger).
See 'handleRemoteCall' and 'handleRemoteTHCall'.
The old interp lock now just protect the pipe writing (reads don't lock
bc they are done exclusively by drainPipe), it doesn't need nor should
lock the full remote procedure call.
- - - - -
10 changed files:
- compiler/GHC/Runtime/Interpreter.hs
- compiler/GHC/Runtime/Interpreter/JS.hs
- compiler/GHC/Runtime/Interpreter/Process.hs
- compiler/GHC/Runtime/Interpreter/Types.hs
- compiler/GHC/Runtime/Interpreter/Wasm.hs
- compiler/GHC/Tc/Gen/Splice.hs
- libraries/ghci/GHCi/Message.hs
- libraries/ghci/GHCi/Server.hs
- libraries/ghci/ghci.cabal.in
- testsuite/tests/ghci/custom-external-interpreter-commands/Main.hs
Changes:
=====================================
compiler/GHC/Runtime/Interpreter.hs
=====================================
@@ -626,11 +626,9 @@ spawnIServ conf = do
[]
(iservConfOpts conf)
interpPipe <- mkPipeFromHandles rh wh
- lock <- newMVar ()
let process = InterpProcess
{ interpHandle = ph
, interpPipe
- , interpLock = lock
}
pending_frees <- newMVar []
=====================================
compiler/GHC/Runtime/Interpreter/JS.hs
=====================================
@@ -127,11 +127,9 @@ startTHRunnerProcess interp_js settings = do
std_in <- readIORef interp_in
interpPipe <- mkPipeFromHandles rh wh
- lock <- newMVar ()
let proc = InterpProcess
{ interpHandle = hdl
, interpPipe
- , interpLock = lock
}
pure (std_in, proc)
@@ -382,7 +380,7 @@ jsLinkPlan logger tmpfs tmp_dir ar_cache link_cfg cfg inst link_plan = do
-- Haskell server.
server_started <- jsServerStarted <$> readMVar (instJSState (instExtra inst))
if server_started
- then sendMessageNoResponse inst $ LoadObj all_js
+ then sendMessage inst $ LoadObj all_js
else jsLoadFile inst all_js
----------------------------------------------------------------
=====================================
compiler/GHC/Runtime/Interpreter/Process.hs
=====================================
@@ -2,16 +2,8 @@ module GHC.Runtime.Interpreter.Process
(
-- * Message API
Message(..)
- , DelayedResponse (..)
- -- * Top-level message API (these acquire/release a lock)
, sendMessage
- , sendMessageNoResponse
- , sendMessageDelayedResponse
- , receiveDelayedResponse
- -- * Nested message API (these require the interpreter to already be locked)
- , sendAnyValue
- , receiveAnyValue
- , receiveTHMessage
+ , sendTHComputation
)
where
@@ -27,97 +19,61 @@ import GHC.Utils.Exception as Ex
import Data.Binary
import System.Exit
import System.Process
-import Control.Concurrent.MVar (MVar, withMVar, takeMVar, putMVar, isEmptyMVar)
-
-data DelayedResponse a = DelayedResponse
+import Control.Concurrent
+import Data.IORef
+import Control.Concurrent.STM
-- -----------------------------------------------------------------------------
-- Top-level Message API
--- | Send a message to the interpreter process that doesn't expect a response
--- (locks the interpreter while sending)
-sendMessageNoResponse :: ExtInterpInstance d -> Message () -> IO ()
-sendMessageNoResponse i m =
- withLock i $ writeInterpProcess (instProcess i) (putMessage m)
-
--- | Send a message to the interpreter that expects a response
--- (locks the interpreter while until the response is received)
+-- | Send a message to the interpreter and expect a response, blocking until it arrives.
+--
+-- This issues an asynchronous RPC on the external interpreter which does not
+-- block the external interpreter thread. We block here on the host because we
+-- wait for the asynchronous response to arrive.
sendMessage :: Binary a => ExtInterpInstance d -> Message a -> IO a
-sendMessage i m = withLock i $ callInterpProcess (instProcess i) m
-
--- | Send a message to the interpreter process whose response is expected later
+sendMessage i msg =
+ remoteCall (interpPipe proc) msg
+ `catchException` \(e :: SomeException) -> handleInterpProcessFailure proc e
+ where
+ proc = instProcess i
+
+-- | Send a message with a TH computation to the ext. interpreter (like
+-- 'RunTH', or 'RunModFinalizers') and handle 'THMessage' requests from the
+-- ext. interpreter to us in the meantime.
--
--- This is useful to avoid forgetting to receive the value and to ensure that
--- the type of the response isn't lost. Use receiveDelayedResponse to read it.
--- (locks the interpreter until the response is received using
--- `receiveDelayedResponse`)
-sendMessageDelayedResponse :: ExtInterpInstance d -> Message a -> IO (DelayedResponse a)
-sendMessageDelayedResponse i m = do
- lock i
- writeInterpProcess (instProcess i) (putMessage m)
- pure DelayedResponse
-
--- | Expect a delayed result to be received now
-receiveDelayedResponse :: Binary a => ExtInterpInstance d -> DelayedResponse a -> IO a
-receiveDelayedResponse i DelayedResponse = do
- ensureLocked i
- r <- readInterpProcess (instProcess i) get
- unlock i
- pure r
-
--- -----------------------------------------------------------------------------
--- Nested Message API
-
--- | Send any value (requires locked interpreter)
-sendAnyValue :: Binary a => ExtInterpInstance d -> a -> IO ()
-sendAnyValue i m = ensureLocked i >> writeInterpProcess (instProcess i) (put m)
-
--- | Expect a value to be received (requires locked interpreter)
-receiveAnyValue :: ExtInterpInstance d -> Get a -> IO a
-receiveAnyValue i get = ensureLocked i >> readInterpProcess (instProcess i) get
-
--- | Wait for a Template Haskell message (requires locked interpreter)
-receiveTHMessage :: ExtInterpInstance d -> IO THMsg
-receiveTHMessage i = ensureLocked i >> receiveAnyValue i getTHMessage
-
--- -----------------------------------------------------------------------------
-
-getLock :: ExtInterpInstance d -> MVar ()
-getLock = interpLock . instProcess
-
-withLock :: ExtInterpInstance d -> IO a -> IO a
-withLock i f = withMVar (getLock i) (const f)
-
-lock :: ExtInterpInstance d -> IO ()
-lock i = takeMVar (getLock i)
-
-unlock :: ExtInterpInstance d -> IO ()
-unlock i = putMVar (getLock i) ()
-
-ensureLocked :: ExtInterpInstance d -> IO ()
-ensureLocked i =
- isEmptyMVar (getLock i) >>= \case
- False -> panic "ensureLocked: external interpreter not locked"
- _ -> pure ()
-
-
--- | Send a 'Message' and receive the response from the interpreter process
-callInterpProcess :: Binary a => InterpProcess -> Message a -> IO a
-callInterpProcess i msg =
- remoteCall (interpPipe i) msg
- `catchException` \(e :: SomeException) -> handleInterpProcessFailure i e
-
--- | Read a value from the interpreter process
-readInterpProcess :: InterpProcess -> Get a -> IO a
-readInterpProcess i get =
- readPipe (interpPipe i) get
- `catchException` \(e :: SomeException) -> handleInterpProcessFailure i e
-
--- | Send a value to the interpreter process
-writeInterpProcess :: InterpProcess -> Put -> IO ()
-writeInterpProcess i put =
- writePipe (interpPipe i) put
- `catchException` \(e :: SomeException) -> handleInterpProcessFailure i e
+-- This function blocks until the TH computation returns a value, but spawns in
+-- the background a thread to handle the 'THMessage's until the TH computation
+-- is fully done.
+sendTHComputation
+ :: Binary a
+ => ExtInterpInstance d
+ -> Message a
+ -> (forall b. Binary b => THMessage b -> IO (Bool, b))
+ -- ^ Handle one message. Return True to end the loop.
+ -> IO a
+sendTHComputation i msg reqHandler = mask $ \restore -> do
+ -- inline "withAsync/wait"
+ var <- newEmptyTMVarIO
+ ht <- forkFinally (restore handlerLoop) (atomically . putTMVar var)
+ restore (sendMessage i msg <* atomically (readTMVar var))
+ `finally` cancel ht var
+ where
+ proc = instProcess i
+ handleOne doneRef = do
+ handleRemoteTHCall (interpPipe proc) $ \req -> do
+ (stop, res) <- reqHandler req
+ writeIORef doneRef stop
+ return res
+ handlerLoop = do
+ doneRef <- newIORef False
+ let loop = do
+ handleOne doneRef
+ isDone <- readIORef doneRef
+ if isDone then return ()
+ else loop
+ in loop
+ cancel ht var = uninterruptibleMask_ (throwTo ht CancelTHHandler <* atomically (readTMVar var))
handleInterpProcessFailure :: InterpProcess -> SomeException -> IO a
handleInterpProcessFailure i e = do
@@ -130,3 +86,8 @@ handleInterpProcessFailure i e = do
terminateProcess hdl
_ <- waitForProcess hdl
throw e
+
+data CancelTHHandler = CancelTHHandler deriving Show
+instance Exception CancelTHHandler where
+ fromException = asyncExceptionFromException
+ toException = asyncExceptionToException
=====================================
compiler/GHC/Runtime/Interpreter/Types.hs
=====================================
@@ -111,7 +111,6 @@ type WasmInterp = ExtInterpState WasmInterpConfig ()
data InterpProcess = InterpProcess
{ interpPipe :: !Pipe -- ^ Pipe to communicate with the server
, interpHandle :: !ProcessHandle -- ^ Process handle of the server
- , interpLock :: !(MVar ()) -- ^ Lock to prevent concurrent access to the stream
}
-- | Status of an external interpreter
=====================================
compiler/GHC/Runtime/Interpreter/Wasm.hs
=====================================
@@ -73,14 +73,12 @@ spawnWasmInterp WasmInterpConfig {..} = do
hSetBuffering rh NoBuffering
interpPipe <- mkPipeFromHandles rh wh
pending_frees <- newMVar []
- lock <- newMVar ()
pure
$ ExtInterpInstance
{ instProcess =
InterpProcess
{ interpHandle = ph,
- interpPipe,
- interpLock = lock
+ interpPipe
},
instPendingFrees = pending_frees,
instExtra = ()
=====================================
compiler/GHC/Tc/Gen/Splice.hs
=====================================
@@ -170,6 +170,7 @@ import Data.IORef
import GHC.Parser.HaddockLex (lexHsDoc)
import GHC.Parser (parseIdentifier)
import GHC.Rename.Doc (rnHsDoc)
+import Data.Function
{-
Note [Template Haskell state diagram]
@@ -1165,11 +1166,12 @@ runRemoteModFinalizers (ThModFinalizers finRefs) = do
case th_state of
Nothing -> return () -- TH was not started, nothing to do
Just fhv -> do
- r <- liftIO $ withForeignRef fhv $ \st ->
- withForeignRefs finRefs $ \qrefs ->
- sendMessageDelayedResponse inst (RunModFinalizers st qrefs)
- () <- runRemoteTH inst []
- qr <- liftIO $ receiveDelayedResponse inst r
+ qr <- withRunInIO $ \runInIO -> do
+ recovers <- newIORef []
+ withForeignRef fhv $ \st ->
+ withForeignRefs finRefs $ \qrefs ->
+ sendTHComputation inst (RunModFinalizers st qrefs) $ \msg ->
+ runInIO (runRemoteTH recovers msg)
checkQResult qr
runQResult
@@ -1824,48 +1826,47 @@ runTH ty fhv = do
rstate <- getTHState inst
loc <- location
-- run a remote TH request
- r <- liftIO $
+ qr <- withRunInIO $ \runInIO -> do
+ recovers <- newIORef []
withForeignRef rstate $ \state_hv ->
- withForeignRef fhv $ \q_hv ->
- sendMessageDelayedResponse inst (RunTH state_hv q_hv ty (Just loc))
- -- respond to requests from the interpreter
- runRemoteTH inst []
- -- get the final result
- qr <- liftIO $ receiveDelayedResponse inst r
+ withForeignRef fhv $ \q_hv ->
+ sendTHComputation inst (RunTH state_hv q_hv ty (Just loc)) $ \msg ->
+ runInIO (runRemoteTH recovers msg)
bs <- checkQResult qr
return $! runGet get (LB.fromStrict bs)
--- | communicate with a remotely-running TH computation until it finishes.
+-- | Communicate with a remotely-running TH computation until it finishes.
-- See Note [Remote Template Haskell] in libraries/ghci/GHCi/TH.hs.
runRemoteTH
- :: ExtInterpInstance d
- -> [Messages TcRnMessage] -- saved from nested calls to qRecover
- -> TcM ()
-runRemoteTH inst recovers = do
- THMsg msg <- liftIO $ receiveTHMessage inst
+ :: Binary a
+ => IORef [Messages TcRnMessage] -- ^ saved from nested calls to qRecover
+ -> THMessage a
+ -> TcM (Bool, a)
+runRemoteTH recovers msg = do
case msg of
- RunTHDone -> return ()
+ RunTHDone -> return (True, ())
StartRecover -> do -- Note [TH recover with -fexternal-interpreter]
v <- getErrsVar
msgs <- readTcRef v
writeTcRef v emptyMessages
- runRemoteTH inst (msgs : recovers)
+ modifyIORef recovers (msgs:) & liftIO
+ return (False, ())
EndRecover caught_error -> do
- let (prev_msgs, rest) = case recovers of
- [] -> panic "EndRecover"
- a : b -> (a,b)
+ (prev_msgs, rest) <- liftIO $
+ readIORef recovers >>= \case
+ [] -> panic "EndRecover"
+ a : b -> pure (a,b)
v <- getErrsVar
warn_msgs <- getWarningMessages <$> readTcRef v
-- keep the warnings only if there were no errors
writeTcRef v $ if caught_error
then prev_msgs
else mkMessages warn_msgs `unionMessages` prev_msgs
- runRemoteTH inst rest
+ writeIORef recovers rest & liftIO
+ return (False, ())
_other -> do
- r <- handleTHMessage msg
- liftIO $ sendAnyValue inst r
- runRemoteTH inst recovers
+ (False,) <$> handleTHMessage msg
-- | Check a QResult
checkQResult :: QResult a -> TcM a
=====================================
libraries/ghci/GHCi/Message.hs
=====================================
@@ -1,6 +1,7 @@
-{-# LANGUAGE GADTs, DeriveGeneric, StandaloneDeriving, ScopedTypeVariables,
- GeneralizedNewtypeDeriving, ExistentialQuantification, RecordWildCards,
- CPP, NamedFieldPuns #-}
+{-# LANGUAGE RecursiveDo, GADTs, DeriveGeneric, StandaloneDeriving, ScopedTypeVariables,
+ GeneralizedNewtypeDeriving, ExistentialQuantification, RecordWildCards, CPP,
+ NamedFieldPuns, LambdaCase, TypeAbstractions, TypeApplications,
+ LinearTypes, RankNTypes, UnboxedTuples, BlockArguments, TupleSections, MagicHash #-}
{-# OPTIONS_GHC -fno-warn-name-shadowing -fno-warn-orphans #-}
-- |
@@ -22,10 +23,25 @@ module GHCi.Message
, ResumeContext(..)
, QState(..)
, getMessage, putMessage, getTHMessage, putTHMessage
- , Pipe, mkPipeFromHandles, mkPipeFromContinuations, remoteCall, remoteTHCall, readPipe, writePipe
+
+ , Pipe(..), mkPipeFromHandles, mkPipeFromContinuations, closePipe
+ , remoteCall, remoteTHCall
+ , handleRemoteCall, handleRemoteTHCall
+ , RemoteProcessException(..)
+
, BreakModule
, BreakUnitId
, LoadedDLL
+
+ -- * Low-level interface for async bidirectional RPC
+ , remoteAsyncReq
+ , handleAsyncReq
+ -- ** Consume `reply` exactly once
+ , WriteReply -- abstract
+ , writeReply
+ -- ** Async matching internals
+ , AsyncManager(..), CorrelationId
+ , IddMsg(..), newAsyncManager
) where
import Prelude -- See note [Why do we import Prelude here?]
@@ -45,6 +61,7 @@ import qualified GHC.Exts.Heap as Heap
import GHC.ForeignSrcLang
import GHC.Fingerprint
import GHC.Conc (pseq, par)
+import GHC.IO (IO(..))
import Control.Concurrent
import Control.DeepSeq
import Control.Exception
@@ -63,6 +80,8 @@ import Data.Dynamic
import Data.Typeable (TypeRep)
import Data.IORef
import Data.Map (Map)
+import Data.IntMap (IntMap)
+import qualified Data.IntMap as IntMap
import Foreign
import GHC.Generics
import GHC.Stack.CCS
@@ -71,6 +90,11 @@ import qualified GHC.Boot.TH.Monad as TH
import System.Exit
import System.IO
import System.IO.Error
+import Control.Monad
+import Control.Concurrent.STM
+import Data.Bifunctor
+import GHC.Base (State#)
+import GHC.Exts (RealWorld)
-- -----------------------------------------------------------------------------
-- The RPC protocol between GHC and the interactive server
@@ -693,42 +717,229 @@ serializeBCOs rbcos = parMap doChunk (chunkList 100 rbcos)
-- | An opaque pipe for bidirectional binary data transmission.
data Pipe = Pipe
- { getSome :: !(IO ByteString)
- , putAll :: !(B.Builder -> IO ())
- , pipeLeftovers :: !(IORef (Maybe ByteString))
+ { getSome :: !(IO ByteString)
+ , putAll :: !(B.Builder -> IO ())
+ , pipeLeftovers :: !(IORef (Maybe ByteString))
+ , pipeLock :: !(MVar ()) -- ^ Lock to prevent concurrent writes to the stream
+ , pipeAsyncMngr :: !AsyncManager
+ , pipeCleanupRW :: !(IO ())
+ -- ^ Clean up the read/write resources, e.g. closes the handles if 'mkPipeFromHandles'
}
-- | Make a 'Pipe' from a 'Handle' to read and a 'Handle' to write.
+-- Must call 'closePipe' before closing the handles, otherwise 'hClose' will
+-- deadlock with the thread blocked draining the read end.
mkPipeFromHandles :: Handle -> Handle -> IO Pipe
mkPipeFromHandles pipeRead pipeWrite = do
let getSome = B.hGetSome pipeRead (32*1024)
putAll b = do
B.hPutBuilder pipeWrite b
hFlush pipeWrite
- pipeLeftovers <- newIORef Nothing
- pure $ Pipe { getSome, putAll, pipeLeftovers }
+ mkPipeFromContinuations getSome putAll (hClose pipeRead >> hClose pipeWrite)
+
+-- | Make a 'Pipe' from a reader function and a writer function, plus an action
+-- to clean up the reader and writer resources. The pipe should be closed with
+-- 'closePipe', which will call this clean up action too.
+mkPipeFromContinuations :: IO ByteString -> (B.Builder -> IO ()) -> IO () -> IO Pipe
+mkPipeFromContinuations getSome putAll pipeCleanupRW = mdo
+ pipeLeftovers <- newIORef Nothing
+ pipeLock <- newMVar ()
+ mdo
+ pipeAsyncMngr <- newAsyncManager p
+ let p = Pipe { .. }
+ pure p
+
+-- | Cleanly close this pipe, terminating the async manager's drain-read-end thread.
+-- This doesn't close the handles
+closePipe :: Pipe -> IO ()
+closePipe Pipe{..} = do
+ killThread (asyncDrainTid pipeAsyncMngr) -- releases handle lock taken by reading
+ pipeCleanupRW -- hClose won't deadlock now that the drainThread is killed
+
+-- | Send a 'Message' request to and wait for a reply from the interpreter
+-- (blocks on the calling side, doesn't necessarily block the interpreter)
+remoteCall :: Binary a => Pipe -> Message a -> IO a
+remoteCall pipe msg = either throwIO (pure . runGet get)
+ =<< remoteAsyncReq pipe (putMessage msg)
+
+-- | Handle a single 'Message' request on the interpreter and reply to the host
+handleRemoteCall :: Pipe
+ -> (Msg -> IO Msg) {-^ Msg hook -}
+ -> (forall a. (Show a, Binary a) => Message a -> IO a)
+ -> IO ()
+handleRemoteCall pipe hook k =
+ handleAsyncReq pipe getMessage \writer msg0 -> IO \s0 ->
+ case hook msg0 |#> s0 of -- unIO for linearity checking
+ (# s1, Msg msg #) -> case k msg |#> s1 of
+ (# s2, a #) -> writeReply writer (put a) |#> s2
+
+-- | Send a 'THMessage' request to the host and wait for a reply from the host
+-- to the interpreter
+remoteTHCall :: Binary a => Pipe -> THMessage a -> IO a
+remoteTHCall pipe msg = either throwIO (pure . runGet get)
+ =<< remoteAsyncReq pipe (putTHMessage msg)
+
+-- | Handle a 'THMessage' request on the host and reply to the interpreter.
+handleRemoteTHCall
+ :: Pipe -> (forall a. (Show a, Binary a) => THMessage a -> IO a) -> IO ()
+handleRemoteTHCall pipe k =
+ handleAsyncReq pipe getTHMessage \writer (THMsg msg) -> IO \s0 ->
+ case k msg |#> s0 of -- unIO for linearity checking
+ (# s1, a #) -> writeReply writer (put a) |#> s1
+
+--------------------------------------------------------------------------------
+-- * Matching requests with async replies
+--------------------------------------------------------------------------------
+
+-- | Send an asynchronous request and block waiting for a matching reply
+remoteAsyncReq :: Pipe -> Put -> IO (Either SomeException LB.ByteString)
+remoteAsyncReq pipe msg = do
+
+ -- Mark pending request
+ uq <- freshReqId (pipeAsyncMngr pipe)
+ wv <- newEmptyMVar
+ atomicModifyIORef' (pendingReply (pipeAsyncMngr pipe)) $
+ \m -> (IntMap.insert uq wv m, ())
+
+ -- Write request
+ writePipe pipe (Request uq (runPut msg))
+
+ -- Block waiting for reply
+ takeMVar wv
+
+-- | Receive a single asynchronous request and send a matching reply
+-- asynchronously (the handler may fork a computation that eventually replies
+-- and can return before actually replying, as long as it eventually does).
+--
+-- The handler receives as an argument the function to use to reply to the request.
+-- It must be used exactly once (all requests expect a reply, and you can't
+-- reply more than once). Linear types enforce the handler replies in every
+-- branch, and if the handler otherwise throws, it is handled by 'withAsyncHandler'.
+handleAsyncReq :: Pipe -> Get a -> (WriteReply Put %1 -> a -> IO r) -> IO r
+handleAsyncReq pipe getit k = do
+ ack <- newEmptyMVar
+
+ -- Block waiting for request
+ (uq, bs) <- atomically (readTQueue (requestsQueue (pipeAsyncMngr pipe)))
+
+ let reply p = do
+ writePipe pipe (Reply uq (bimap toSerializableException runPut p))
+ putMVar ack ()
+
+ -- Run the continuation which must reply exactly once with 'writeReply'
+ k (WriteReply reply) (runGet getit bs)
+
+ -- If there's an exception before the reply is sent, reply with an error instead.
+ `catch` \(e :: SomeException) -> do
+
+ tryTakeMVar ack >>= \case
+ Just () ->
+ -- Already replied
+ throwIO e
+ Nothing -> do
+ -- Reply with an error
+ reply (Left e)
+ throwIO e
+
+-- | An exception that can be thrown on the host process representing an
+-- expection that happened on the remote one.
+newtype RemoteProcessException = RemoteProcessException SerializableException deriving Show
+instance Exception RemoteProcessException
+
+-- ** Async matching "internals" -----------------------------------------------
+
+data AsyncManager = AsyncManager
+ { pendingReply :: !(IORef (IntMap (MVar (Either SomeException LB.ByteString))))
+ -- ^ When a request is sent, keep track that we're expecting a reply. The
+ -- MVar is signaled when the reply arrives. The exception can originate
+ -- from the host process, or from the remote one (with 'RemoteProcessException').
+ , requestsQueue :: !(TQueue (CorrelationId, LB.ByteString))
+ -- ^ A queue of pending received requests. When a request is received,
+ -- queue it here. A request handler will pop the next request from it.
+ -- (Nothing except the 'drainPipe' thread should read from the pipe).
+ , asyncDrainTid :: !ThreadId
+ , asyncNextReq :: !(IORef Int)
+ }
--- | Make a 'Pipe' from a reader function and a writer function.
-mkPipeFromContinuations :: IO ByteString -> (B.Builder -> IO ()) -> IO Pipe
-mkPipeFromContinuations getSome putAll = do
- pipeLeftovers <- newIORef Nothing
- pure $ Pipe { getSome, putAll, pipeLeftovers }
+type CorrelationId = Int
-remoteCall :: Binary a => Pipe -> Message a -> IO a
-remoteCall pipe msg = do
- writePipe pipe (putMessage msg)
- readPipe pipe get
+-- | We have a single type for requests and replies because both the host and
+-- interpreter may send requests *and* replies on the write end *and* on recv
+-- both on the read end.
+--
+-- For instance:
+-- 1. The host sends a RunTH request on its Host Write End (HWE)
+-- 2. The interpreter sends a NewName TH request on its Interpreter Write End (IWE)
+-- 3. The host sends a reply to the TH request on its HWE
+-- 4. The interpreter sends the reply to the RunTH request on its IWE
+data IddMsg = Request !CorrelationId LB.ByteString
+ | Reply !CorrelationId (Either SerializableException LB.ByteString)
+
+instance Binary IddMsg where
+ put (Request i bs) = putWord8 0 >> put i >> put bs
+ put (Reply i bs) = putWord8 1 >> put i >> put bs
+ get = do
+ b <- getWord8
+ case b of
+ 0 -> Request <$> get <*> get
+ 1 -> Reply <$> get <*> get
+ _ -> error $ "impossible: Binary IddMsg - " ++ show b
-writePipe :: Pipe -> Put -> IO ()
-writePipe Pipe{..} put = putAll $ execPut put
+newAsyncManager :: Pipe -> IO AsyncManager
+newAsyncManager pipe = do
+ asyncNextReq <- newIORef 0
+ pendingReply <- newIORef IntMap.empty
+ requestsQueue <- newTQueueIO
-remoteTHCall :: Binary a => Pipe -> THMessage a -> IO a
-remoteTHCall pipe msg = do
- writePipe pipe (putTHMessage msg)
- readPipe pipe get
+ mdo
+ -- Fork thread to drain read end.
+ -- (cleaned up on 'closePipe')
+ asyncDrainTid <- forkIO (drainPipe pipe am)
+
+ let am = AsyncManager{..}
+ pure am
-readPipe :: Pipe -> Get a -> IO a
-readPipe Pipe{..} get = do
+-- | Drains the read end of the Pipe and match 'Reply'es against pending 'Request's.
+--
+-- In pratice, we'll have both a thread draining the read end of the host's
+-- 'Pipe' and a thread draining the read end of the interpreter's 'Pipe', since
+-- both processes construct a 'Pipe' and send and receive messages:
+drainPipe :: Pipe -> AsyncManager -> IO ()
+drainPipe p am = flip catch cancelPending $ forever $ do
+ readPipe p >>= \case
+ -- Receive a reply to a pending notification
+ Reply uq bs -> do
+ wv <- atomicModifyIORef' (pendingReply am) $ \m -> do
+ case IntMap.lookup uq m of
+ Nothing -> error "drainPipe: Received reply to no pending request"
+ Just wv -> (IntMap.delete uq m, wv)
+ putMVar wv (bimap (toException . RemoteProcessException) id bs) -- signal matching request
+ -- Receive a request and queue it
+ Request uq bs ->
+ atomically $ writeTQueue (requestsQueue am) (uq,bs)
+ where
+ cancelPending (e::SomeException) = do
+ pendings <- readIORef (pendingReply am)
+ mapM_ (\p -> putMVar p (Left e {-host exception-})) pendings
+
+freshReqId :: AsyncManager -> IO CorrelationId
+freshReqId AsyncManager{asyncNextReq} =
+ atomicModifyIORef' asyncNextReq $ \i -> (i+1, i)
+
+--------------------------------------------------------------------------------
+-- * Pipe thread-safe read/write
+--------------------------------------------------------------------------------
+-- All messages on the wire are 'IddMsg', which contain a correlation header
+-- and identify whether they are requests or replies.
+
+writePipe :: Pipe -> IddMsg -> IO ()
+writePipe p@Pipe{..} = withLock p . putAll . execPut . put
+
+-- | Only the 'drainPipe' thread should read from the pipe directly. Other
+-- functions should wait for the matching reply to be delivered asynchronously.
+-- (see 'withAsyncReq'). It doesn't lock to read bc it assumes single reader is 'drainPipe'.
+readPipe :: Pipe -> IO IddMsg
+readPipe Pipe{..} = do
leftovers <- readIORef pipeLeftovers
m <- getBin getSome get leftovers
case m of
@@ -758,3 +969,22 @@ getBin getsome get leftover = go leftover (runGetIncremental get)
else go Nothing (fun (Just b))
go _lft (Fail _rest _off str) =
throwIO (ErrorCall ("getBin: " ++ str))
+
+withLock :: Pipe -> IO c -> IO c
+withLock Pipe{..} = bracket (takeMVar pipeLock) (putMVar pipeLock) . const
+
+--------------------------------------------------------------------------------
+-- * Consume `reply` exactly once
+--------------------------------------------------------------------------------
+
+-- | Every request should get exactly one reply.
+data WriteReply a where
+ WriteReply :: (Either SomeException a -> IO ()) %1 -> WriteReply a
+
+-- | Write a reply exactly once (consuming the ability to write more replies)
+writeReply :: WriteReply a %1 -> a -> IO ()
+writeReply (WriteReply r) = \x -> r (Right x)
+
+-- | Infix, linear, 'unIO'
+(|#>) :: IO a %1 -> State# RealWorld -> (# State# RealWorld, a #)
+(|#>) (IO fs) s = fs s
=====================================
libraries/ghci/GHCi/Server.hs
=====================================
@@ -1,4 +1,4 @@
-{-# LANGUAGE CPP, RankNTypes, RecordWildCards, GADTs, ScopedTypeVariables #-}
+{-# LANGUAGE LambdaCase, CPP, RankNTypes, RecordWildCards, GADTs, ScopedTypeVariables #-}
module GHCi.Server
( MessageHook
, CustomMessageHandler
@@ -15,6 +15,7 @@ import GHCi.Signals
import GHCi.TH
import GHCi.Message
import Data.ByteString (ByteString)
+import Data.IORef
#if defined(wasm32_HOST_ARCH)
import qualified Data.ByteString.Builder as B
import qualified Data.ByteString.Internal as B
@@ -60,27 +61,26 @@ servWithCustom
-> (forall a .IO a -> IO a)
-> CustomMessageHandler
-> IO ()
-servWithCustom verbose hook pipe restore customHandler = loop
+servWithCustom verbose hook pipe restore customHandler = newIORef False >>= loop
where
- loop = do
+ loop doneRef = do
when verbose $ trace "reading pipe..."
- Msg msg <- readPipe pipe getMessage >>= hook
+ handleRemoteCall pipe hook $ \(msg :: Message a) -> do
- discardCtrlC
+ discardCtrlC
- when verbose $ trace ("msg: " ++ (show msg))
- case msg of
- CustomMessage tag payload -> handleCustom tag payload
- Shutdown -> return ()
- RunTH st q ty loc -> wrapRunTH $ runTH pipe st q ty loc
- RunModFinalizers st qrefs -> wrapRunTH $ runModFinalizerRefs pipe st qrefs
- _other -> run msg >>= reply
+ when verbose $ trace ("msg: " ++ (show msg))
+ case msg of
+ CustomMessage tag payload -> handleCustom tag payload
+ Shutdown -> writeIORef doneRef True >> reply ()
+ -- TODO: These two should probably be done off the main thread
+ RunTH st q ty loc -> wrapRunTH $ runTH pipe st q ty loc
+ RunModFinalizers st qrefs -> wrapRunTH $ runModFinalizerRefs pipe st qrefs
+ _other -> run msg >>= reply
- reply :: forall a. (Binary a, Show a) => a -> IO ()
- reply r = do
- when verbose $ trace ("writing pipe: " ++ show r)
- writePipe pipe (put r)
- loop
+ isDone <- readIORef doneRef
+ if isDone then return ()
+ else loop doneRef
handleCustom tag payload = do
mresp <- customHandler tag payload
@@ -89,17 +89,22 @@ servWithCustom verbose hook pipe restore customHandler = loop
Nothing ->
error $ "GHCi.Server: unhandled CustomMessage with tag " ++ show tag
+ reply :: forall a. Show a => a -> IO a
+ reply r = do
+ when verbose $ trace ("writing pipe: " ++ show r)
+ return r -- the result of the handler gets sent
+
-- Run some TH code, which may interact with GHC by sending
-- THMessage requests, and then finally send RunTHDone followed by a
-- QResult. For an overview of how TH works with Remote GHCi, see
-- Note [Remote Template Haskell] in libraries/ghci/GHCi/TH.hs.
- wrapRunTH :: forall a. (Binary a, Show a) => IO a -> IO ()
+ wrapRunTH :: forall a. (Binary a, Show a) => IO a -> IO (QResult a)
wrapRunTH io = do
when verbose $ trace "wrapRunTH..."
r <- try io
when verbose $ trace "wrapRunTH done."
when verbose $ trace "writing RunTHDone."
- writePipe pipe (putTHMessage RunTHDone)
+ remoteTHCall pipe RunTHDone
case r of
Left e
| Just (GHCiQException _ err) <- fromException e -> do
@@ -179,7 +184,7 @@ defaultServerWithCustom customHandler = do
#if defined(wasm32_HOST_ARCH)
-- See Note [wasm ghci signal handlers] for details
installSignalHandlers $ js_register_signal_handler cb_sig
- pipe <- mkPipeFromContinuations (recv_buf cb_recv) (send_buf cb_send)
+ pipe <- mkPipeFromContinuations (recv_buf cb_recv) (send_buf cb_send) (pure ())
#else
when verbose $
printf "GHC iserv starting (in: %s; out: %s)\n" (show inh) (show outh)
=====================================
libraries/ghci/ghci.cabal.in
=====================================
@@ -95,7 +95,8 @@ library
filepath >= 1.4 && < 1.6,
ghc-boot == @ProjectVersionMunged@,
ghc-heap >= 9.10.1 && <=@ProjectVersionMunged@,
- transformers >= 0.5 && < 0.7
+ transformers >= 0.5 && < 0.7,
+ stm >= 2.4 && < 2.6
if flag(bootstrap)
build-depends:
=====================================
testsuite/tests/ghci/custom-external-interpreter-commands/Main.hs
=====================================
@@ -12,6 +12,7 @@ import GHCi.Message
, mkPipeFromHandles
, remoteCall
, Pipe
+ , closePipe
)
import GHCi.Server
( CustomMessageHandler
@@ -153,14 +154,13 @@ customMessage pipe c = do
runClient :: Int -> IO ()
runClient input = do
serverExe <- getExecutablePath
- withServer serverExe $ \hFromServer hToServer -> do
- pipe <- mkPipeFromHandles hFromServer hToServer
+ withServer serverExe $ \pipe -> do
res <- customMessage pipe (SquareCommand input)
putStrLn $ "Square returned: " ++ show res
res2 <- customMessage pipe (MulCommand 2 res)
putStrLn $ "Mul returned: " ++ show res2
-withServer :: FilePath -> (Handle -> Handle -> IO a) -> IO a
+withServer :: FilePath -> (Pipe -> IO a) -> IO a
withServer serverExe action = do
(ghcRead, serverWrite) <- createPipe
(serverRead, ghcWrite) <- createPipe
@@ -171,13 +171,12 @@ withServer serverExe action = do
, std_out = Inherit
, std_err = Inherit
}
- bracket (mkHandles ghcRead ghcWrite)
- (\(hFromServer, hToServer) -> do
- hClose hFromServer
- hClose hToServer
+ bracket (mkHandles ghcRead ghcWrite >>= uncurry mkPipeFromHandles)
+ (\pipe -> do
+ closePipe pipe
terminateProcess ph
void (waitForProcess ph))
- (\(hFromServer, hToServer) -> action hFromServer hToServer)
+ (\pipe -> action pipe)
where
mkHandles r w = do
hR <- fdToHandle r
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/daab9334a12d96543ede9ac61e14b00…
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/daab9334a12d96543ede9ac61e14b00…
You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help
1
0
[Git][ghc/ghc][wip/andrea/with-signal-handlers] add runGhcWithSignalHandlers and remove handling from runGhc[T]
by Andrea Vezzosi (@trac-Saizan) 13 Oct '26
by Andrea Vezzosi (@trac-Saizan) 13 Oct '26
13 Oct '26
Andrea Vezzosi pushed to branch wip/andrea/with-signal-handlers at Glasgow Haskell Compiler / GHC
Commits:
1091494a by Andrea Vezzosi at 2026-10-13T12:45:34+02:00
add runGhcWithSignalHandlers and remove handling from runGhc[T]
it only ever makes sense to call `withSignalHandlers` from `main`, otherwise
signals can be too easily swallowed without notification to the rest of your
process.
This commit takes withSignalHandlers out of `runGhc/runGhcT` and provides
`runGhcWithSignalHandlers` for the old behaviour.
- - - - -
14 changed files:
- + changelog.d/T4162
- compiler/GHC.hs
- compiler/GHC/Utils/Panic.hs
- ghc/Main.hs
- linters/lint-codes/LintCodes/Static.hs
- + testsuite/tests/ghc-api/T4162.hs
- + testsuite/tests/ghc-api/T4162.stdout
- testsuite/tests/ghc-api/all.T
- utils/check-exact/Parsers.hs
- utils/check-ppr/Main.hs
- utils/count-deps/Main.hs
- utils/dump-decls/Main.hs
- utils/haddock/haddock-api/src/Haddock.hs
- utils/haddock/haddock-api/test/Haddock/Backends/Hyperlinker/ParserSpec.hs
Changes:
=====================================
changelog.d/T4162
=====================================
@@ -0,0 +1,17 @@
+section: compiler
+issues: #4162
+mrs: !16774
+synopsis:
+ `runGhc/runGhcT` no longer hijack process signals, old behaviour provided by
+ `runGhcWithSignalHandlers`.
+description:
+ Functions `runGhc/runGhcT` no longer go out of their way to redirect process
+ signals as exceptions to them.
+
+ The installed handlers make it so other threads are unaware of any signals,
+ which can cause the process to become unresponsive whenever a crash of the GHC
+ session(s) does not cascade into a shutdown of the process itself.
+
+ The old behaviour is available through `runGhcWithSignalHandlers` or by
+ calling `withSignalHandlers`. Only intended to be used from the main thread of
+ your process, see the documentation.
=====================================
compiler/GHC.hs
=====================================
@@ -23,6 +23,7 @@ module GHC (
-- * GHC Monad
Ghc, GhcT, GhcMonad(..), HscEnv,
runGhc, runGhcT, initGhcMonad,
+ runGhcWithSignalHandlers,
printException,
handleSourceError,
@@ -555,10 +556,26 @@ runGhc :: Maybe FilePath -- ^ See argument to 'initGhcMonad'.
runGhc mb_top_dir ghc = do
ref <- newIORef (panic "empty session")
let session = Session ref
- flip unGhc session $ withSignalHandlers $ do -- catch ^C
+ flip unGhc session $ do
initGhcMonad mb_top_dir
withCleanupSession ghc
+-- | Like 'runGhc' but uses 'withSignalHandlers' to redirect selected signals to
+-- this thread to trigger 'withCleanupSession', and so that interactive
+-- evaluation receives 'UserInterrupt' exceptions.
+--
+-- __WARNING__: Only intended to be called from the __main thread__ of your
+-- process, see 'withSignalHandlers' documentation for more details.
+--
+-- More complex multi-threaded programs are recommended to use 'runGhc' or
+-- 'runGhcT', and handle signals as appropriate for their control flow.
+
+runGhcWithSignalHandlers :: Maybe FilePath -- ^ See argument to 'initGhcMonad'.
+ -> Ghc a -- ^ The action to perform.
+ -> IO a
+runGhcWithSignalHandlers mb_top_dir ghc = runGhc mb_top_dir $
+ withSignalHandlers $ ghc
+
-- | Run function for 'GhcT' monad transformer.
--
-- It initialises the GHC session and warnings via 'initGhcMonad'. Each call
@@ -572,7 +589,7 @@ runGhcT :: ExceptionMonad m =>
runGhcT mb_top_dir ghct = do
ref <- liftIO $ newIORef (panic "empty session")
let session = Session ref
- flip unGhcT session $ withSignalHandlers $ do -- catch ^C
+ flip unGhcT session $ do
initGhcMonad mb_top_dir
withCleanupSession ghct
=====================================
compiler/GHC/Utils/Panic.hs
=====================================
@@ -234,8 +234,21 @@ signalHandlersRefCount :: MVar (Word, Maybe S.Handler)
signalHandlersRefCount = unsafePerformIO $ newMVar (0,Nothing)
--- | Temporarily install standard signal handlers for catching ^C, which just
--- throw an exception in the current thread.
+-- | Temporarily install signal handlers to redirect the following signals to
+-- the calling thread as async exceptions: @INT@, @QUIT@, @HUP@, @TERM@ on unix, and
+-- @CtrlC@, @Break@ on windows. The original signal handlers are restored when all
+-- concurrent calls of this function are exited.
+--
+-- @INT@, @QUIT@, @CtrlC@, and @Break@ are thrown as @'UserInterrupt'@.
+--
+-- @HUP@, and @TERM@ are thrown as @'Panic' <signal number>@.
+--
+-- __WARNING__: Only intended to be called from the __main thread__ of your
+-- process, as otherwise it can swallow signals the process should react to in
+-- other ways. Nested calls from a single thread are fine, but concurrent calls
+-- from different threads will have the exceptions delivered to only one of
+-- them. If that thread is not @main@, extra care is needed to e.g. actually
+-- terminate the process when it receives @TERM@.
withSignalHandlers :: ExceptionMonad m => m a -> m a
#if !defined(HAVE_SIGNAL_H)
-- No signal functionality exist on the host platform (e.g. on
=====================================
ghc/Main.hs
=====================================
@@ -141,7 +141,7 @@ main = do
ShowOptions isInteractive -> showOptions isInteractive
Right postStartupMode ->
-- start our GHC session
- GHC.runGhc mbMinusB $ do
+ GHC.runGhcWithSignalHandlers mbMinusB $ do
dflags <- GHC.getSessionDynFlags
=====================================
linters/lint-codes/LintCodes/Static.hs
=====================================
@@ -31,7 +31,7 @@ import GHC.Types.Error.Codes
-- ghc (API usage)
import GHC
- ( runGhc, parseDynamicFlags
+ ( runGhcWithSignalHandlers, parseDynamicFlags
, getSessionDynFlags, setSessionDynFlags
, getSession, getLogger
, noLoc
@@ -145,7 +145,7 @@ newtype LibDir = LibDir { libDir :: FilePath }
-- | Look up the 'GhcDiagnosticCode' type family using the GHC API.
ghcDiagnosticCodeTyCon :: Maybe LibDir -> IO TyCon
ghcDiagnosticCodeTyCon mb_libDir =
- runGhc (libDir <$> mb_libDir)
+ runGhcWithSignalHandlers (libDir <$> mb_libDir)
-- STEP 1: start a GHC API session with "-package ghc"
do { dflags1 <- getSessionDynFlags
=====================================
testsuite/tests/ghc-api/T4162.hs
=====================================
@@ -0,0 +1,145 @@
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE LambdaCase #-}
+
+-- | Test for #4162: 'runGhc' must not install signal handlers on behalf of its
+-- caller.
+--
+-- 'withSignalHandlers' redirects signals to the thread that called it. While
+-- 'runGhc' did that implicitly, a GHC session running in a 'forkIO' thread
+-- would capture signals aimed at the process as a whole and deliver them to a
+-- thread that may not be in a position to act on them. Here that thread sits
+-- under 'uninterruptibleMask', so the exception can never be delivered and the
+-- signal is swallowed outright.
+--
+-- The worker process parks a 'forkIO' thread inside 'runGhc' under
+-- 'uninterruptibleMask' and blocks its main thread waiting for that thread to
+-- finish. The driver process waits for the worker to report that the thread is
+-- inside 'runGhc', signals it, and checks that it dies promptly:
+--
+-- * on unix, SIGTERM has no handler, so the kernel kills the process;
+-- * on Windows, CTRL_BREAK is handled by GHC.TopHandler, which throws
+-- UserInterrupt at the main thread, which exits with 252.
+--
+-- Before #4162 both were swallowed by the masked 'forkIO' thread and the worker
+-- carried on running.
+module Main (main) where
+
+import Control.Concurrent
+import Control.Exception
+import Control.Monad
+import Control.Monad.IO.Class (liftIO)
+import GHC (runGhc)
+import System.Environment
+import System.Exit
+import System.IO
+import System.Process
+import System.Timeout (timeout)
+
+#if !defined(mingw32_HOST_OS)
+import System.Posix.Signals (sigKILL, sigTERM, signalProcess)
+#endif
+
+main :: IO ()
+main = getArgs >>= \case
+ [libdir] -> driver libdir
+ [libdir, "--worker"] -> worker libdir
+ args -> die ("FAILED: unexpected arguments: " ++ show args)
+
+-- | The process under test: a GHC session pinned in a 'forkIO' thread, with the
+-- main thread waiting on it.
+worker :: FilePath -> IO ()
+worker libdir = do
+ hSetBuffering stdout LineBuffering
+ done <- newEmptyMVar
+ _ <- forkIO $
+ flip finally (putMVar done ()) $
+ uninterruptibleMask_ $
+ runGhc (Just libdir) $ liftIO $ do
+ putStrLn ready
+ forever (threadDelay oneSecond)
+ takeMVar done
+ -- Only reachable if the session thread died on its own; the signal was
+ -- supposed to take the whole process down first.
+ putStrLn "worker: session thread finished"
+
+-- | Wait for the worker to reach 'runGhc', signal it, and require a prompt exit.
+driver :: FilePath -> IO ()
+driver libdir = do
+ self <- getExecutablePath
+ let cp = (proc self [libdir, "--worker"])
+ { std_out = CreatePipe
+#if defined(mingw32_HOST_OS)
+ -- GenerateConsoleCtrlEvent can only target a process group, so the
+ -- worker needs one of its own.
+ , create_group = True
+#endif
+ }
+ bracket (createProcess cp) kill $ \(_, Just out, _, ph) -> do
+ handshake <- timeout handshakeTimeout
+ (try (hGetLine out) :: IO (Either IOException String))
+ case handshake of
+ Just (Right l) | l == ready -> return ()
+ _ -> die ("FAILED: worker never reached runGhc: " ++ show handshake)
+ interruptWorker ph
+ waitForExit ph >>= \case
+ Just code | code == expectedExit -> putStrLn "worker terminated promptly"
+ other -> die ("FAILED: expected " ++ show expectedExit
+ ++ " from the worker, got " ++ show other)
+
+-- | Deliver the signal that the worker must not be able to swallow.
+interruptWorker :: ProcessHandle -> IO ()
+#if defined(mingw32_HOST_OS)
+-- CTRL_BREAK_EVENT rather than CTRL_C_EVENT: GenerateConsoleCtrlEvent cannot
+-- deliver CTRL+C to a specific process group. Both reach the same handler, in
+-- GHC.TopHandler now and in withSignalHandlers before #4162.
+interruptWorker = interruptProcessGroupOf
+#else
+interruptWorker ph = getPid ph >>= \case
+ Just pid -> signalProcess sigTERM pid
+ Nothing -> die "FAILED: worker exited before it could be signalled"
+#endif
+
+-- | How the worker is expected to die once signalled.
+expectedExit :: ExitCode
+#if defined(mingw32_HOST_OS)
+-- UserInterrupt reaches the top handler, which calls exitInterrupted.
+expectedExit = ExitFailure 252
+#else
+expectedExit = ExitFailure (negate (fromIntegral sigTERM))
+#endif
+
+-- | Poll for the worker's exit, giving up after 'terminationTimeout'. Polling
+-- rather than 'waitForProcess' so that a worker that ignored the signal fails
+-- the test quickly instead of hanging it until the testsuite timeout.
+waitForExit :: ProcessHandle -> IO (Maybe ExitCode)
+waitForExit ph = go (terminationTimeout `div` pollInterval)
+ where
+ go :: Int -> IO (Maybe ExitCode)
+ go n
+ | n <= 0 = return Nothing
+ | otherwise = getProcessExitCode ph >>= \case
+ Just code -> return (Just code)
+ Nothing -> threadDelay pollInterval >> go (n - 1)
+
+-- | Keep a worker that ignored the signal from outliving the test and holding
+-- the inherited stderr open. Note that 'terminateProcess' is only SIGTERM on
+-- unix, which is precisely what such a worker swallows.
+kill :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> IO ()
+kill (_, _, _, ph) = do
+#if defined(mingw32_HOST_OS)
+ terminateProcess ph
+#else
+ getPid ph >>= mapM_ (signalProcess sigKILL)
+#endif
+ void (waitForProcess ph)
+
+-- | Printed by the worker from inside 'runGhc', so that by the time the driver
+-- sees it any handlers 'runGhc' might install are already in place.
+ready :: String
+ready = "ready"
+
+oneSecond, pollInterval, handshakeTimeout, terminationTimeout :: Int
+oneSecond = 1000000
+pollInterval = 100000 -- 0.1s
+handshakeTimeout = 60 * oneSecond -- starting a GHC session can be slow
+terminationTimeout = 10 * oneSecond
=====================================
testsuite/tests/ghc-api/T4162.stdout
=====================================
@@ -0,0 +1 @@
+worker terminated promptly
=====================================
testsuite/tests/ghc-api/all.T
=====================================
@@ -91,3 +91,13 @@ test('T27240', [extra_run_opts(f'"{config.libdir}"')], compile_and_run, ['-packa
test('T27273', [extra_run_opts(f'"{config.libdir}"')],
compile_and_run,
['-package ghc'])
+
+# runGhc must not install signal handlers on behalf of its caller, or a session
+# running off the main thread makes the process unresponsive to signals.
+test('T4162', [ extra_run_opts(f'"{config.libdir}"')
+ # the test re-executes itself, so it has to be a real executable
+ , only_ways(['normal'])
+ , req_process
+ # no subprocesses or signals
+ , when(arch('wasm32') or arch('javascript'), skip)
+ ], compile_and_run, ['-package ghc'])
=====================================
utils/check-exact/Parsers.hs
=====================================
@@ -231,7 +231,7 @@ parseModuleEpAnnsWithCpp libdir cppOptions file = ghcWrapper libdir $ do
ghcWrapper :: LibDir -> GHC.Ghc a -> IO a
ghcWrapper libdir a =
GHC.defaultErrorHandler GHC.defaultFatalMessager GHC.defaultFlushOut
- $ GHC.runGhc (Just libdir) a
+ $ GHC.runGhcWithSignalHandlers (Just libdir) a
-- | Internal function. Exposed if you want to muck with DynFlags
-- before parsing.
=====================================
utils/check-ppr/Main.hs
=====================================
@@ -81,7 +81,7 @@ testOneFile libdir fileName = do
parseOneFile :: FilePath -> FilePath -> IO ParsedModule
parseOneFile libdir fileName = do
- runGhc (Just libdir) $ do
+ runGhcWithSignalHandlers (Just libdir) $ do
dflags <- getSessionDynFlags
let dflags2 = dflags `gopt_set` Opt_KeepRawTokenStream
_ <- setSessionDynFlags dflags2
=====================================
utils/count-deps/Main.hs
=====================================
@@ -51,7 +51,7 @@ printDeps libdir modName dot = do
calcDeps :: String -> FilePath -> IO (Map.Map ModuleName [ModuleName])
calcDeps modName libdir =
defaultErrorHandler defaultFatalMessager defaultFlushOut $ do
- runGhc (Just libdir) $ do
+ runGhcWithSignalHandlers (Just libdir) $ do
df <- getSessionDynFlags
logger <- getLogger
(df, _, _) <- parseDynamicFlags logger df [noLoc "-package=ghc"]
=====================================
utils/dump-decls/Main.hs
=====================================
@@ -32,7 +32,7 @@ main = do
mapM_ (run ghcRoot) pkg_names
run :: FilePath -> String -> IO ()
-run root pkg_nm = runGhc (Just root) $ do
+run root pkg_nm = runGhcWithSignalHandlers (Just root) $ do
let args = map noLoc
[ "-package=" ++ pkg_nm
, "-dppr-cols=1000"
=====================================
utils/haddock/haddock-api/src/Haddock.hs
=====================================
@@ -196,7 +196,7 @@ handleGhcExceptions =
--
-- > main = getArgs >>= haddock
haddock :: [String] -> IO ()
-haddock args = haddockWithGhc withGhc args
+haddock args = withSignalHandlers $ haddockWithGhc withGhc args
haddockWithGhc :: (forall a. [Flag] -> Ghc a -> IO a) -> [String] -> IO ()
haddockWithGhc ghc args = handleTopExceptions $ do
=====================================
utils/haddock/haddock-api/test/Haddock/Backends/Hyperlinker/ParserSpec.hs
=====================================
@@ -20,7 +20,7 @@ import Haddock.Backends.Hyperlinker.Types
withDynFlags :: (DynFlags -> IO ()) -> IO ()
withDynFlags cont = do
libDir <- fmap snd (getGhcDirs [])
- runGhc libDir $ do
+ runGhcWithSignalHandlers libDir $ do
dflags <- getSessionDynFlags
liftIO $ cont dflags
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/1091494a093356c0c10d70e88ddeebb…
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/1091494a093356c0c10d70e88ddeebb…
You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help
1
0