PTH-- Remove feature entirely-

When debugging a boost build with a modified
version of Clang, I discovered that the PTH implementation
stores TokenKind in 8 bits. However, we currently have 368
TokenKinds.

The result is that the value gets truncated and the wrong token
gets picked up when including PTH files. It seems that this will
go wrong every time someone uses a token that uses the 9th bit.

Upon asking on IRC, it was brought up that this was a highly
experimental features that was considered a failure. I discovered
via googling that BoostBuild (mostly Boost.Math) is the only user of
this
feature, using the CC1 flag directly. I believe that this can be
transferred over to normal PCH with minimal effort:
boostorg/build#367

Based on advice on IRC and research showing that this is a nearly
completely unused feature, this patch removes it entirely.

Note: I considered leaving the build-flags in place and making them
emit an error/warning, however since I've basically identified and
warned the only user, it seemed better to just remove them.

Differential Revision: https://reviews.llvm.org/D54547

Change-Id: If32744275ef1f585357bd6c1c813d96973c4d8d9
llvm-svn: 348266

Author: Erich Keane

clang/NOTES.txt

@@ -8,13 +8,6 @@ To time GCC preprocessing speed without output, use:
    "time gcc -MM file"
 This is similar to -Eonly.
 
-//===---------------------------------------------------------------------===//
-
-Creating and using a PTH file for performance measurement (use a release build).
-
-$ clang -ccc-pch-is-pth -x objective-c-header INPUTS/Cocoa_h.m -o /tmp/tokencache
-$ clang -cc1 -token-cache /tmp/tokencache INPUTS/Cocoa_h.m
-
 //===---------------------------------------------------------------------===//
 
   C++ Template Instantiation benchmark:

clang/docs/InternalsManual.rst

@@ -559,13 +559,9 @@ The clang Driver and library are documented :doc:`here <DriverInternals>`.
 Precompiled Headers
 ===================
 
-Clang supports two implementations of precompiled headers.  The default
-implementation, precompiled headers (:doc:`PCH <PCHInternals>`) uses a
+Clang supports precompiled headers (:doc:`PCH <PCHInternals>`), which  uses a
 serialized representation of Clang's internal data structures, encoded with the
 `LLVM bitstream format <https://llvm.org/docs/BitCodeFormat.html>`_.
-Pretokenized headers (:doc:`PTH <PTHInternals>`), on the other hand, contain a
-serialized representation of the tokens encountered when preprocessing a header
-(and anything that header includes).
 
 The Frontend Library
 ====================

clang/docs/PTHInternals.rst

@@ -116,7 +116,9 @@ Improvements to Clang's diagnostics
 Non-comprehensive list of changes in this release
 -------------------------------------------------
 
-- ...
+- The experimental feature Pretokenized Headers (PTH) was removed in its
+  entirely from Clang. The feature did not properly work with about 1/3 of the
+  possible tokens available and was unmaintained.
 
 New Compiler Flags
 ------------------

clang/docs/ReleaseNotes.rst

@@ -99,8 +99,6 @@ def err_fe_unable_to_interface_with_target : Error<
     "unable to interface with target machine">;
 def err_fe_unable_to_open_output : Error<
     "unable to open output file '%0': '%1'">;
-def err_fe_pth_file_has_no_source_header : Error<
-    "PTH file '%0' does not designate an original source header file for -include-pth">;
 def warn_fe_macro_contains_embedded_newline : Warning<
     "macro '%0' contains embedded newline; text after the newline is ignored">;
 def warn_fe_cc_print_header_failure : Warning<

clang/include/clang/Basic/DiagnosticFrontendKinds.td

@@ -258,13 +258,6 @@ def warn_bad_character_encoding : ExtWarn<
 def err_lexing_string : Error<"failure when lexing a string">;
 def err_placeholder_in_source : Error<"editor placeholder in source file">;
 
-
-//===----------------------------------------------------------------------===//
-// PTH Diagnostics
-//===----------------------------------------------------------------------===//
-def err_invalid_pth_file : Error<
-    "invalid or corrupt PTH file '%0'">;
-
 //===----------------------------------------------------------------------===//
 // Preprocessor Diagnostics
 //===----------------------------------------------------------------------===//
@@ -570,8 +563,6 @@ def err_pp_module_end_without_module_begin : Error<
   "'#pragma clang module end'">;
 def note_pp_module_begin_here : Note<
   "entering module '%0' due to this pragma">;
-def err_pp_module_build_pth : Error<
-  "'#pragma clang module build' not supported in pretokenized header">;
 def err_pp_module_build_missing_end : Error<
   "no matching '#pragma clang module endbuild' for this '#pragma clang module build'">;
 

clang/include/clang/Basic/DiagnosticLexKinds.td

@@ -562,8 +562,6 @@ def emit_module_interface : Flag<["-"], "emit-module-interface">,
   HelpText<"Generate pre-compiled module file from a C++ module interface">;
 def emit_header_module : Flag<["-"], "emit-header-module">,
   HelpText<"Generate pre-compiled module file from a set of header files">;
-def emit_pth : Flag<["-"], "emit-pth">,
-  HelpText<"Generate pre-tokenized header file">;
 def emit_pch : Flag<["-"], "emit-pch">,
   HelpText<"Generate pre-compiled header file">;
 def emit_llvm_bc : Flag<["-"], "emit-llvm-bc">,
@@ -794,15 +792,11 @@ def internal_externc_isystem : JoinedOrSeparate<["-"], "internal-externc-isystem
 // Preprocessor Options
 //===----------------------------------------------------------------------===//
 
-def include_pth : Separate<["-"], "include-pth">, MetaVarName<"<file>">,
-  HelpText<"Include file before parsing">;
 def chain_include : Separate<["-"], "chain-include">, MetaVarName<"<file>">,
   HelpText<"Include and chain a header file after turning it into PCH">;
 def preamble_bytes_EQ : Joined<["-"], "preamble-bytes=">,
   HelpText<"Assume that the precompiled header is a precompiled preamble "
            "covering the first N bytes of the main file">;
-def token_cache : Separate<["-"], "token-cache">, MetaVarName<"<path>">,
-  HelpText<"Use specified token cache file">;
 def detailed_preprocessing_record : Flag<["-"], "detailed-preprocessing-record">,
   HelpText<"include a detailed record of preprocessing actions">;
 

clang/include/clang/Driver/CC1Options.td

@@ -227,9 +227,6 @@ class Driver {
   unsigned CheckInputsExist : 1;
 
 public:
-  /// Use lazy precompiled headers for PCH support.
-  unsigned CCCUsePCH : 1;
-
   /// Force clang to emit reproducer for driver invocation. This is enabled
   /// indirectly by setting FORCE_CLANG_DIAGNOSTICS_CRASH environment variable
   /// or when using the -gen-reproducer driver flag.

clang/include/clang/Driver/Driver.h

@@ -258,10 +258,6 @@ def rsp_quoting : Joined<["--"], "rsp-quoting=">, Group<internal_driver_Group>,
 def ccc_gcc_name : Separate<["-"], "ccc-gcc-name">, InternalDriverOpt,
   HelpText<"Name for native GCC compiler">,
   MetaVarName<"<gcc-path>">;
-def ccc_pch_is_pch : Flag<["-"], "ccc-pch-is-pch">, InternalDriverOpt,
-  HelpText<"Use lazy PCH for precompiled headers">;
-def ccc_pch_is_pth : Flag<["-"], "ccc-pch-is-pth">, InternalDriverOpt,
-  HelpText<"Use pretokenized headers for precompiled headers">;
 
 class InternalDebugOpt : Group<internal_debug_Group>,
   Flags<[DriverOption, HelpHidden, CoreOption]>;

clang/include/clang/Driver/Options.td

@@ -261,11 +261,6 @@ class DumpTokensAction : public PreprocessorFrontendAction {
   void ExecuteAction() override;
 };
 
-class GeneratePTHAction : public PreprocessorFrontendAction {
-protected:
-  void ExecuteAction() override;
-};
-
 class PreprocessOnlyAction : public PreprocessorFrontendAction {
 protected:
   void ExecuteAction() override;

clang/include/clang/Frontend/FrontendActions.h

@@ -88,9 +88,6 @@ enum ActionKind {
   /// Generate pre-compiled header.
   GeneratePCH,
 
-  /// Generate pre-tokenized header.
-  GeneratePTH,
-
   /// Only execute frontend initialization.
   InitOnly,
 

clang/include/clang/Frontend/FrontendOptions.h

@@ -183,9 +183,6 @@ void AttachHeaderIncludeGen(Preprocessor &PP,
                             StringRef OutputPath = {},
                             bool ShowDepth = true, bool MSStyle = false);
 
-/// Cache tokens for use with PCH. Note that this requires a seekable stream.
-void CacheTokens(Preprocessor &PP, raw_pwrite_stream *OS);
-
 /// The ChainedIncludesSource class converts headers to chained PCHs in
 /// memory, mainly for testing.
 IntrusiveRefCntPtr<ExternalSemaSource>