Merge remote-tracking branch 'gkgl/bug_40019_v3'

  cp /var/tmp/dist/mingw-w64/gcclibs/{libssp.a,libssp_nonshared.a} /var/tmp/dist/mingw-w64-clang/[% c("arch") %]-w64-mingw32/lib/

[% END -%]

[% IF c("var/osx") %]

  mkdir -p /var/tmp/dist/macosx-toolchain/clang/lib/clang/[% pc("clang", "version") %]/lib/darwin

  mv libclang_rt.osx.a /var/tmp/dist/macosx-toolchain/clang/lib/clang/[% pc("clang", "version") %]/lib/darwin

[% END -%]

tar -C /var/tmp/dist -xf [% c('input_files_by_name/rust') %]

tar -C /var/tmp/dist -xf [% c('input_files_by_name/cbindgen') %]

tar -C /var/tmp/dist -xf [% c('input_files_by_name/nasm') %]

    # TorButton patch authored by Arthur Edelstein, from https://github.com/arthuredelstein/torbutton/ branch 2.1.10-namecoin

  - filename: namecoin-etld.patch

    enable: '[% c("var/namecoin") %]'

    # The library is extracted from Mozilla's officially used Clang, downloaded

    # from https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/TYTqgyybSLmp0O0GcIl8Xw/artifacts/public/build/clang.tar.zst.

    # Its SHA-256 sum is 7b0a6e60f102d3dbf618ae9e25d990d4dc72c76eb5accb5974ec70c11ff7e049.

  - URL: https://people.torproject.org/~gk/mirrors/sources/libclang_rt.osx.a

    sha256sum: 33541fb054e7dc372b5e9708bbcc175561d7fdbc9b00bf964e0c60b82bd8165a

    enable: '[% c("var/osx") %]'

From c95310f2d4fd3c88241c3b5d6dbf6251d34a3256 Mon Sep 17 00:00:00 2001

From: Nikita Popov <nikita.ppv@gmail.com>

Date: Sat, 16 Nov 2019 16:22:18 +0100

Subject: [PATCH] Restructure caching

Variant on D70103. The caching is switched to always use a BB to

cache entry map, which then contains per-value caches. A separate

set contains value handles with a deletion callback. This allows us

to properly invalidate overdefined values.

A possible alternative would be to always cache by value first and

have per-BB maps/sets in the each cache entry. In that case we could

use a ValueMap and would avoid the separate value handle set. I went

with the BB indexing at the top level to make it easier to integrate

D69914, but possibly that's not the right choice.

Differential Revision: https://reviews.llvm.org/D70376

diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp

index 110c085d3f3..aa6862cb588 100644

--- a/llvm/lib/Analysis/LazyValueInfo.cpp

+++ b/llvm/lib/Analysis/LazyValueInfo.cpp

@@ -133,12 +133,9 @@ namespace {

   /// A callback value handle updates the cache when values are erased.

   class LazyValueInfoCache;

   struct LVIValueHandle final : public CallbackVH {

-    // Needs to access getValPtr(), which is protected.

-    friend struct DenseMapInfo<LVIValueHandle>;

-

     LazyValueInfoCache *Parent;

-    LVIValueHandle(Value *V, LazyValueInfoCache *P)

+    LVIValueHandle(Value *V, LazyValueInfoCache *P = nullptr)

       : CallbackVH(V), Parent(P) { }

     void deleted() override;

@@ -152,89 +149,63 @@ namespace {

   /// This is the cache kept by LazyValueInfo which

   /// maintains information about queries across the clients' queries.

   class LazyValueInfoCache {

-    /// This is all of the cached block information for exactly one Value*.

-    /// The entries are sorted by the BasicBlock* of the

-    /// entries, allowing us to do a lookup with a binary search.

-    /// Over-defined lattice values are recorded in OverDefinedCache to reduce

-    /// memory overhead.

-    struct ValueCacheEntryTy {

-      ValueCacheEntryTy(Value *V, LazyValueInfoCache *P) : Handle(V, P) {}

-      LVIValueHandle Handle;

-      SmallDenseMap<PoisoningVH<BasicBlock>, ValueLatticeElement, 4> BlockVals;

+    /// This is all of the cached information for one basic block. It contains

+    /// the per-value lattice elements, as well as a separate set for

+    /// overdefined values to reduce memory usage.

+    struct BlockCacheEntryTy {

+      SmallDenseMap<AssertingVH<Value>, ValueLatticeElement, 4> LatticeElements;

+      SmallDenseSet<AssertingVH<Value>, 4> OverDefined;

     };

-    /// This tracks, on a per-block basis, the set of values that are

-    /// over-defined at the end of that block.

-    typedef DenseMap<PoisoningVH<BasicBlock>, SmallPtrSet<Value *, 4>>

-        OverDefinedCacheTy;

-    /// Keep track of all blocks that we have ever seen, so we

-    /// don't spend time removing unused blocks from our caches.

-    DenseSet<PoisoningVH<BasicBlock> > SeenBlocks;

-

-    /// This is all of the cached information for all values,

-    /// mapped from Value* to key information.

-    DenseMap<Value *, std::unique_ptr<ValueCacheEntryTy>> ValueCache;

-    OverDefinedCacheTy OverDefinedCache;

-

+    /// Cached information per basic block.

+    DenseMap<PoisoningVH<BasicBlock>, BlockCacheEntryTy> BlockCache;

+    /// Set of value handles used to erase values from the cache on deletion.

+    DenseSet<LVIValueHandle, DenseMapInfo<Value *>> ValueHandles;

   public:

     void insertResult(Value *Val, BasicBlock *BB,

                       const ValueLatticeElement &Result) {

-      SeenBlocks.insert(BB);

-

+      auto &CacheEntry = BlockCache.try_emplace(BB).first->second;

       // Insert over-defined values into their own cache to reduce memory

       // overhead.

       if (Result.isOverdefined())

-        OverDefinedCache[BB].insert(Val);

-      else {

-        auto It = ValueCache.find_as(Val);

-        if (It == ValueCache.end()) {

-          ValueCache[Val] = make_unique<ValueCacheEntryTy>(Val, this);

-          It = ValueCache.find_as(Val);

-          assert(It != ValueCache.end() && "Val was just added to the map!");

-        }

-        It->second->BlockVals[BB] = Result;

-      }

-    }

-

-    bool isOverdefined(Value *V, BasicBlock *BB) const {

-      auto ODI = OverDefinedCache.find(BB);

-

-      if (ODI == OverDefinedCache.end())

-        return false;

+        CacheEntry.OverDefined.insert(Val);

+      else

+        CacheEntry.LatticeElements.insert({ Val, Result });

-      return ODI->second.count(V);

+      auto HandleIt = ValueHandles.find_as(Val);

+      if (HandleIt == ValueHandles.end())

+        ValueHandles.insert({ Val, this });

     }

     bool hasCachedValueInfo(Value *V, BasicBlock *BB) const {

-      if (isOverdefined(V, BB))

-        return true;

-

-      auto I = ValueCache.find_as(V);

-      if (I == ValueCache.end())

+      auto It = BlockCache.find(BB);

+      if (It == BlockCache.end())

         return false;

-      return I->second->BlockVals.count(BB);

+      return It->second.OverDefined.count(V) ||

+             It->second.LatticeElements.count(V);

     }

     ValueLatticeElement getCachedValueInfo(Value *V, BasicBlock *BB) const {

-      if (isOverdefined(V, BB))

+      auto It = BlockCache.find(BB);

+      if (It == BlockCache.end())

+        return ValueLatticeElement();

+

+      if (It->second.OverDefined.count(V))

         return ValueLatticeElement::getOverdefined();

-      auto I = ValueCache.find_as(V);

-      if (I == ValueCache.end())

+      auto LatticeIt = It->second.LatticeElements.find(V);

+      if (LatticeIt == It->second.LatticeElements.end())

         return ValueLatticeElement();

-      auto BBI = I->second->BlockVals.find(BB);

-      if (BBI == I->second->BlockVals.end())

-        return ValueLatticeElement();

-      return BBI->second;

+

+      return LatticeIt->second;

     }

     /// clear - Empty the cache.

     void clear() {

-      SeenBlocks.clear();

-      ValueCache.clear();

-      OverDefinedCache.clear();

+      BlockCache.clear();

+      ValueHandles.clear();

     }

     /// Inform the cache that a given value has been deleted.

@@ -248,23 +219,18 @@ namespace {

     /// OldSucc might have (unless also overdefined in NewSucc).  This just

     /// flushes elements from the cache and does not add any.

     void threadEdgeImpl(BasicBlock *OldSucc,BasicBlock *NewSucc);

-

-    friend struct LVIValueHandle;

   };

 }

 void LazyValueInfoCache::eraseValue(Value *V) {

-  for (auto I = OverDefinedCache.begin(), E = OverDefinedCache.end(); I != E;) {

-    // Copy and increment the iterator immediately so we can erase behind

-    // ourselves.

-    auto Iter = I++;

-    SmallPtrSetImpl<Value *> &ValueSet = Iter->second;

-    ValueSet.erase(V);

-    if (ValueSet.empty())

-      OverDefinedCache.erase(Iter);

+  for (auto &Pair : BlockCache) {

+    Pair.second.LatticeElements.erase(V);

+    Pair.second.OverDefined.erase(V);

   }

-  ValueCache.erase(V);

+  auto HandleIt = ValueHandles.find_as(V);

+  if (HandleIt != ValueHandles.end())

+    ValueHandles.erase(HandleIt);

 }

 void LVIValueHandle::deleted() {

@@ -274,18 +240,7 @@ void LVIValueHandle::deleted() {

 }

 void LazyValueInfoCache::eraseBlock(BasicBlock *BB) {

-  // Shortcut if we have never seen this block.

-  DenseSet<PoisoningVH<BasicBlock> >::iterator I = SeenBlocks.find(BB);

-  if (I == SeenBlocks.end())

-    return;

-  SeenBlocks.erase(I);

-

-  auto ODI = OverDefinedCache.find(BB);

-  if (ODI != OverDefinedCache.end())

-    OverDefinedCache.erase(ODI);

-

-  for (auto &I : ValueCache)

-    I.second->BlockVals.erase(BB);

+  BlockCache.erase(BB);

 }

 void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc,

@@ -303,10 +258,11 @@ void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc,

   std::vector<BasicBlock*> worklist;

   worklist.push_back(OldSucc);

-  auto I = OverDefinedCache.find(OldSucc);

-  if (I == OverDefinedCache.end())

+  auto I = BlockCache.find(OldSucc);

+  if (I == BlockCache.end() || I->second.OverDefined.empty())

     return; // Nothing to process here.

-  SmallVector<Value *, 4> ValsToClear(I->second.begin(), I->second.end());

+  SmallVector<Value *, 4> ValsToClear(I->second.OverDefined.begin(),

+                                      I->second.OverDefined.end());

   // Use a worklist to perform a depth-first search of OldSucc's successors.

   // NOTE: We do not need a visited list since any blocks we have already

@@ -320,10 +276,10 @@ void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc,

     if (ToUpdate == NewSucc) continue;

     // If a value was marked overdefined in OldSucc, and is here too...

-    auto OI = OverDefinedCache.find(ToUpdate);

-    if (OI == OverDefinedCache.end())

+    auto OI = BlockCache.find(ToUpdate);

+    if (OI == BlockCache.end() || OI->second.OverDefined.empty())

       continue;

-    SmallPtrSetImpl<Value *> &ValueSet = OI->second;

+    auto &ValueSet = OI->second.OverDefined;

     bool changed = false;

     for (Value *V : ValsToClear) {

@@ -333,11 +289,6 @@ void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc,

       // If we removed anything, then we potentially need to update

       // blocks successors too.

       changed = true;

-

-      if (ValueSet.empty()) {

-        OverDefinedCache.erase(OI);

-        break;

-      }

     }

     if (!changed) continue;

-- 

2.24.0

#!/bin/bash

[% c("var/set_default_env") -%]

builddir=/var/tmp/build

mkdir $builddir

distdir="/var/tmp/dist/[% project %]"

mkdir -p "$distdir"

tar xjf [% c('input_files_by_name/SDK') %]

mv MacOSX10.11.sdk "$distdir/"

tar xf [% c('input_files_by_name/clang') %]

mv clang "$distdir/clang"

tar -C /var/tmp/dist -xf [% c('input_files_by_name/cmake') %]

tar -C /var/tmp/dist -xf [% c('input_files_by_name/ninja') %]

tar -C /var/tmp/dist -xf [% c('input_files_by_name/clang_compiler') %]

tar -C $builddir -xf [% c('input_files_by_name/llvm') %]

tar -C $builddir -xf [% c('input_files_by_name/clang') %]

tar -C $builddir -xf [% c('input_files_by_name/libcxx') %]

tar -C $builddir -xf [% c('input_files_by_name/libcxxabi') %]

tar -C $builddir -xf [% c('input_files_by_name/lld') %]

tar -C $builddir -xf [% c('input_files_by_name/compiler-rt') %]

tar -C $distdir -xf [% c('input_files_by_name/SDK') %]

tar -C $distdir -xf [% c('input_files_by_name/cctools') %]

cd $distdir/cctools/bin

clangdir=/var/tmp/dist/clang/bin

cctoolsdir=$distdir/cctools/bin

sysrootdir=$distdir/MacOSX10.11.sdk/

# We still need to put the cctoolsdir on the path. That's because of `lipo`. See

# the respective comment in the cctools build script.

export PATH="/var/tmp/dist/ninja:/var/tmp/dist/cmake/bin:$cctoolsdir:$PATH"

export MACOSX_DEPLOYMENT_TARGET=[% c("var/macosx_deployment_target") %]

cd $builddir

mv llvm-* llvm

# LLVM has reproducibility issues when optimizing bitcode, which we need to

# patch. See: #32053 for more details.

patch -p1 < $rootdir/43909.patch

mv clang-* llvm/tools/clang

mv libcxx-* llvm/projects/libcxx

mv libcxxabi-* llvm/projects/libcxxabi

mv lld-* llvm/tools/lld

mv compiler-rt-* compiler-rt

patch -p1 < $rootdir/compiler-rt-cross-compile.patch

patch -p1 < $rootdir/compiler-rt-no-codesign.patch

mv compiler-rt llvm/projects

mkdir build

cd build

# We follow quite closely Mozilla's build/build-clang/build-clang.py with the

# clang-macosx64.json flavor. This currently implies disabling Xray as it

# requires an OSX 10.12 SDK (see: https://bugs.llvm.org/show_bug.cgi?id=38959).

cmake -GNinja \

      -DCMAKE_C_COMPILER=$clangdir/clang \

      -DCMAKE_CXX_COMPILER=$clangdir/clang++ \

      -DCMAKE_ASM_COMPILER=$clangdir/clang \

      -DCMAKE_LINKER=$clangdir/clang \

      -DCMAKE_AR=$cctoolsdir/x86_64-apple-darwin-ar \

      -DCMAKE_C_FLAGS="[% c('var/FLAGS') %] -I$sysrootdir/usr/include -iframework $sysrootdir/System/Library/Frameworks" \

      -DCMAKE_CXX_FLAGS="-stdlib=libc++ [% c('var/FLAGS') %] -I$sysrootdir/usr/include -iframework $sysrootdir/System/Library/Frameworks" \

      -DCMAKE_ASM_FLAGS="[% c('var/FLAGS') %] -I$sysrootdir/usr/include -iframework $sysrootdir/System/Library/Frameworks" \

      -DCMAKE_EXE_LINKER_FLAGS="[% c('var/LDFLAGS') %]" \

      -DCMAKE_SHARED_LINKER_FLAGS="[% c('var/LDFLAGS') %]" \

      -DCMAKE_BUILD_TYPE=Release \

      -DCMAKE_INSTALL_PREFIX=$distdir/clang \

      -DLLVM_TARGETS_TO_BUILD="X86;ARM;AArch64" \

      -DLLVM_ENABLE_ASSERTIONS=OFF \

      -DPYTHON_EXECUTABLE=/usr/bin/python2.7 \

      -DLLVM_TOOL_LIBCXX_BUILD=ON \

      -DLLVM_ENABLE_BINDINGS=OFF \

      -DLLVM_ENABLE_LIBXML2=FORCE_ON \

      -DLLVM_LINK_LLVM_DYLIB=ON \

      -DCMAKE_RANLIB=$cctoolsdir/x86_64-apple-darwin-ranlib \

      -DCMAKE_LIBTOOL=$cctoolsdir/x86_64-apple-darwin-libtool \

      -DCMAKE_SYSTEM_NAME=Darwin \

      -DCMAKE_SYSTEM_VERSION=10.10 \

      -DLLVM_ENABLE_THREADS=OFF \

      -DCOMPILER_RT_BUILD_XRAY=OFF \

      -DLIBCXXABI_LIBCXX_INCLUDES=$builddir/llvm/projects/libcxx/include \

      -DCMAKE_OSX_SYSROOT=$sysrootdir \

      -DCMAKE_FIND_ROOT_PATH=$sysrootdir \

      -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \

      -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \

      -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY \

      -DCMAKE_MACOSX_RPATH=ON \

      -DCMAKE_OSX_ARCHITECTURES=x86_64 \

      -DDARWIN_osx_ARCHS=x86_64 \

      -DDARWIN_osx_SYSROOT=$sysrootdir \

      -DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin $builddir/llvm

ninja install -v

# We now have a native macosx64 toolchain.

# What we want is a native linux64 toolchain which can target macosx64.

# Overlay the linux64 toolchain that we used for this build. Note: we don't do

# any ASan builds for macOS (yet). Once we do we need to preserve at least

# `llvm-symbolizer` as that one seems to get shipped with sanitizer builds (at

# least that's what Mozilla claims).

cd $distdir

cp --remove-destination -lr /var/tmp/dist/clang/* clang/

cd cctools/bin

ln -s ../../clang/bin/clang x86_64-apple-darwin-clang

ln -s ../../clang/bin/clang++ x86_64-apple-darwin-clang++

cd "/var/tmp/dist"

cd /var/tmp/dist

[% c('tar', {

   tar_src => [ project ],

   tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),

Add `-target x86_64-apple-darwin' to the compiler-rt overridden CFLAGS

diff --git a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake

index 28d398672..aac68bf36 100644

--- a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake

+++ b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake

@@ -265,7 +265,7 @@ endfunction()

 macro(darwin_add_builtin_libraries)

   set(DARWIN_EXCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/Darwin-excludes)

-  set(CFLAGS "-fPIC -O3 -fvisibility=hidden -DVISIBILITY_HIDDEN -Wall -fomit-frame-pointer")

+  set(CFLAGS "-fPIC -O3 -fvisibility=hidden -DVISIBILITY_HIDDEN -Wall -fomit-frame-pointer -target x86_64-apple-darwin -isysroot ${CMAKE_OSX_SYSROOT} -I${CMAKE_OSX_SYSROOT}/usr/include")

   set(CMAKE_C_FLAGS "")

   set(CMAKE_CXX_FLAGS "")

   set(CMAKE_ASM_FLAGS "")