fixup

Author: satanson

CMakeLists.txt

@@ -6,10 +6,12 @@ project(cpp_etudes)
 #enable_language(C)
 #enable_language(ASM)
 
-set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -static-libstdc++ -msse4.1 -msse4.2 -mavx2 -mpopcnt -O3 -std=c++17")
-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -static-libstdc++ -msse4.1 -msse4.2 -mavx2 -g -O0 -std=c++17")
+#set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -static-libstdc++ -msse4.1 -msse4.2 -mavx2 -mpopcnt -O3 -std=c++17")
+#set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -static-libstdc++ -msse4.1 -msse4.2 -mavx2 -g -O0 -std=c++17")
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -static-libstdc++ -O3 -std=c++17")
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -static-libstdc++ -g -O0 -std=c++17")
 #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
-
+message(STATUS "abc")
 include(cmakes/arch.cmake)
 include(cmakes/target.cmake)
 #include(cmakes/tools.cmake)
@@ -143,6 +145,20 @@ set(ABSL_BUILD_TESTING OFF)
 add_subdirectory(third-party/abseil-cpp-91b7cd600a34cbd)
 set(BUILD_TESTING ON)
 
+include (CheckCXXSourceCompiles)
+check_cxx_source_compiles("
+#include <iostream>
+int main() {
+#if !_GLIBCXX_USE_CXX11_ABI
+#error _GLIBCXX_USE_CXX11_ABI must be non-zero
+#endif
+ return 0;
+}
+" HAVE_GLIBCXX_USE_CXX11_ABI)
+if (NOT HAVE_GLIBCXX_USE_CXX11_ABI)
+ MESSAGE(FATAL_ERROR "_GLIBCXX_USE_CXX11_ABI should not be 0")
+endif()
+
 list(APPEND link_libs gtest
  pthread atomic
  glog::glog
@@ -170,3 +186,14 @@ message(STATUS "link_libs=${link_libs}")
 add_subdirectory(unittest)
 add_subdirectory(benchmark)
 add_subdirectory(src/main)
+
+SET(IN_DOCKER_ENV OFF)
+FIND_FILE(DOCKER_ENV NAMES ".dockerenv" PATHS "/" NO_CACHE)
+if (NOT (DOCKER_ENV STREQUAL "DOCKER_ENV-NOTFOUND"))
+ SET(IN_DOCKER_ENV ON)
+endif()
+
+
+if (IN_DOCKER_ENV)
+MESSAGE(STATUS "IN_DOCKER_ENV=${IN_DOCKER_ENV}")
+endif()

cmakes/1.png

@@ -2,13 +2,20 @@
 
 use strict;
 use warnings;
+use List::Util qw/max/;
+use Data::Dumper;
 my $n = shift or die "Missing argument 'n'";
+my $sep = shift or die "Missing argument 'sep'";
+my $fill = shift or die "Missing argument 'fill'";
+
 my @ifiles=map {open my $h, "<$_" or die "Open file '$_' error=$!";$h} @ARGV;
 
 my @contents = map{my $h=$_;[map{chomp $_;$_} <$h>]} @ifiles;
-my @total_lines = map{scalar(@$_)} @contents;
+@contents = map {[@{$_}[0..$n-1]] } map{[@$_, ($fill) x $n]} @contents;
+
+#print Dumper(\@contents);
 
 for my $i (0..$n-1) {
- print join "\t", map {$contents[$_][$i%$total_lines[$_]]} 0..$#contents;
+ print join "$sep", map {$contents[$_][$i]} 0..$#contents;
  print "\n";
 }

paste.pl

@@ -23,6 +23,9 @@ ($)
  elsif ($t=~/^(\d+)m(\d+)s$/) {
  return (($1+0)*60+$2)*1000;
  }
+ elsif($t=~/^(\d+)m$/) {
+ return ($1+0)*60*1000;
+ }
  elsif ($t=~/^-(\d+(?:\.\d+)?)(ns|us|ms)$/){
  return -($1+0.0)/$unit{$2};
  }
@@ -53,7 +56,7 @@ ($)
  if (!defined($u)) {
  $u="B";
  }
- return ($1+0.0)*$unit{$u};
+ return ($1+0.0)*$unit{$u}/1024/1024/1024;
  } else {
  die "undefined number format! '$b'";
  return undef;
@@ -84,7 +87,7 @@ ($)
  next;
  }
 
- if (/\b(MemoryLimit|PeakMemoryUsage|InstanceNum):\s+(\S+)/) {
+ if (/\b(MemoryLimit|PeakMemoryUsage|__MAX_OF_PeakMemoryUsage|InstancePeakMemoryUsage|__MAX_OF_InstancePeakMemoryUsage|InstanceNum):\s+(.*)/) {
  $plan->{$fragment}{$1}=$2;
  $plan->{$fragment}{id}=$fragment;
  next;
@@ -115,7 +118,7 @@ ($)
  next;
  }
 
- if (/\b(PushTotalTime|PullTotalTime|CompressTime|SetFinishingTime|BuildHashTableTime|RuntimeFilterBuildTime|CopyRightTableChunkTime|OtherJoinConjunctEvaluateTime|OutputBuildColumnTimer|OutputProbeColumnTimer|OutputTupleColumnTimer|ProbeConjunctEvaluateTime|__MAX_OF_ProbeConjunctEvaluateTime|__MIN_OF_ProbeConjunctEvaluateTime|SearchHashTableTimer|WhereConjunctEvaluateTime|OperatorTotalTime|SetFinishedTime|JoinRuntimeFilterTime|CloseTime)\b:\s+(\S+)/) {
+ if (/\b(PushTotalTime|SortingTime|__MAX_OF_SortingTime|MergingTime|__MAX_OF_MergingTime|PullTotalTime|CompressTime|SetFinishingTime|BuildHashTableTime|RuntimeFilterBuildTime|CopyRightTableChunkTime|OtherJoinConjunctEvaluateTime|OutputBuildColumnTimer|OutputProbeColumnTimer|OutputTupleColumnTimer|ProbeConjunctEvaluateTime|__MAX_OF_ProbeConjunctEvaluateTime|__MIN_OF_ProbeConjunctEvaluateTime|SearchHashTableTimer|WhereConjunctEvaluateTime|OperatorTotalTime|SetFinishedTime|JoinRuntimeFilterTime|CloseTime)\b:\s+(\S+)/) {
  my $operator_id = join "_", ($fragment, $pipeline, $operator);
  $plan->{$fragment}{pipelines}{$pipeline}{operators}{$operator}{$1}=norm_time($2);
  $plan->{$fragment}{pipelines}{$pipeline}{operators}{$operator}{id}=$operator_id;
@@ -128,7 +131,7 @@ ($)
  $plan->{$fragment}{pipelines}{$pipeline}{operators}{$operator}{id}=$operator_id;
  next;
  }
- if (/\b(BytesPassThrough|BytesSent|UpcompressedBytes):\s+(.*)/) {
+ if (/\b(BytesPassThrough|BytesSent|InputRequiredMemory|__MAX_OF_InputRequiredMemory|MergeUnsortedPeakMemoryUsage|__MAX_OF_MergeUnsortedPeakMemoryUsage|MergeSortedPeakMemoryUsage|__MAX_OF_MergeSortedPeakMemoryUsage|SortPartialPeakMemoryUsage|__MAX_OF_SortPartialPeakMemoryUsage|UpcompressedBytes|OperatorPeakMemoryUsage|__MAX_OF_OperatorPeakMemoryUsage):\s+(.*)/) {
  my $operator_id = join "_", ($fragment, $pipeline, $operator);
  $plan->{$fragment}{pipelines}{$pipeline}{operators}{$operator}{$1}=$2;
  $plan->{$fragment}{pipelines}{$pipeline}{operators}{$operator}{id}=$operator_id;
@@ -154,5 +157,5 @@ ($)
 my @fragments = grep {exists $_->{$index}} @$fragments;
 my @pipelines = grep {exists $_->{$index}} @$pipelines;
 my @ops= grep {exists $_->{$index}} @$ops;
-print join "\n", map {sprintf "%s\t%s\t%s", "".$_->{$index}, $index, $_->{id}} sort{$a->{$index} <=> $b->{$index}} (@fragments, @pipelines, @ops);
+print join "\n", map {sprintf "%s\t%s\t%s", "".$_->{$index}, $index, $_->{id}} sort{$a->{$index} cmp $b->{$index}} (@fragments, @pipelines, @ops);
 print "\n";

sr_index2.pl

@@ -1461,6 +1461,21 @@ TEST_F(MiscTest, testReadValues) {
  int e =0;
  ASSERT_FALSE(read_values("foobar.1", a, b, c, s, d, e));
 }
+void* func(void*){
+ std::cout<<"ok"<<std::endl;
+}
+TEST_F(MiscTest, testPthread) {
+ pthread_t pid;
+ pthread_attr_t pattr;
+ pthread_attr_init(&pattr);
+ pthread_attr_setstack(&pattr, (void*)static_cast<intptr_t>(1), 100);
+ auto res = pthread_create(&pid, &pattr, func, NULL);
+ if (res!=0) {
+ std::cout<<"pthread_create: ERROR";
+ }
+ pthread_join(pid,NULL);
+
+}
 
 int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);

unittest/misc_test.cc

@@ -6,9 +6,15 @@
 //
 // Created by grakra on 2022/11/12.
 //
+#include <glog/logging.h>
 #include <gtest/gtest.h>
+#include <llvm-c/BitWriter.h>
 #include <llvm-c/Core.h>
+#include <llvm-c/LLJIT.h>
+#include <llvm-c/OrcEE.h>
+#include <llvm-c/Orc.h>
 #include <llvm-c/TargetMachine.h>
+#include <llvm/ExecutionEngine/Orc/LLJIT.h>
 
 #include <memory>
 namespace test {
@@ -74,6 +80,56 @@ TEST_F(LLVMTest, testTupleReference) {
  }
  std::cout << "end" << std::endl;
 }
+
+static LLVMOrcObjectLayerRef llvm_create_object_layer(void* Ctx, LLVMOrcExecutionSessionRef ES, const char* Triple) {
+ LLVMOrcObjectLayerRef objlayer = LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
+
+#if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
+ if (jit_debugging_support) {
+ LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
+
+ LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
+ }
+#endif
+
+#if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
+ if (jit_profiling_support) {
+ LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
+
+ LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
+ }
+#endif
+
+ return objlayer;
+}
+
+static void llvm_log_jit_error(void* ctx, LLVMErrorRef error) {
+ std::cerr << "Error: " << LLVMGetErrorMessage(error) << std::endl;
+}
+
+static void materializationUnitFn() {}
+static LLVMErrorRef
+definitionGeneratorFn(LLVMOrcDefinitionGeneratorRef G, void *Ctx,
+ LLVMOrcLookupStateRef *LS, LLVMOrcLookupKind K,
+ LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags F,
+ LLVMOrcCLookupSet Names, size_t NamesCount) {
+ for (size_t I = 0; I < NamesCount; I++) {
+ LLVMOrcCLookupSetElement Element = Names[I];
+ LLVMOrcJITTargetAddress Addr =
+ (LLVMOrcJITTargetAddress)(&materializationUnitFn);
+ LLVMJITSymbolFlags Flags = {LLVMJITSymbolGenericFlagsWeak, 0};
+ LLVMJITEvaluatedSymbol Sym = {Addr, Flags};
+ LLVMOrcRetainSymbolStringPoolEntry(Element.Name);
+ LLVMJITCSymbolMapPair Pair = {Element.Name, Sym};
+ LLVMJITCSymbolMapPair Pairs[] = {Pair};
+ LLVMOrcMaterializationUnitRef MU = LLVMOrcAbsoluteSymbols(Pairs, 1);
+ LLVMErrorRef Err = LLVMOrcJITDylibDefine(JD, MU);
+ if (Err)
+ return Err;
+ }
+ return LLVMErrorSuccess;
+}
+
 TEST_F(LLVMTest, testBasic2) {
  auto llvmCtx = LLVMContextCreate();
  ASSERT_TRUE(LLVMInitializeNativeTarget() == 0);
@@ -92,9 +148,68 @@ TEST_F(LLVMTest, testBasic2) {
  LLVMSetDataLayout(module, dataLayout);
  auto builder = LLVMCreateBuilder();
  std::vector<LLVMTypeRef> funcParams(2, LLVMInt32Type());
- auto funcType = LLVMFunctionType(LLVMInt32Type(),funcParams.data(), funcParams.size(), false);
- LLVMAddFunction(module, "foobar", funcType);
+ auto funcType = LLVMFunctionType(LLVMInt32Type(), funcParams.data(), funcParams.size(), false);
+ auto func = LLVMAddFunction(module, "foobar", funcType);
+ LLVMSetLinkage(func, LLVMExternalLinkage);
+ LLVMSetVisibility(func, LLVMDefaultVisibility);
+ auto block0 = LLVMAppendBasicBlock(func, "entry");
+ auto param0 = LLVMGetParam(func, 0);
+ auto param1 = LLVMGetParam(func, 1);
+ LLVMPositionBuilderAtEnd(builder, block0);
+ auto tmp = LLVMBuildAlloca(builder, LLVMInt32Type(), "alloc");
+ auto addResult = LLVMBuildAdd(builder, param0, param1, "add");
+ LLVMBuildStore(builder, addResult, tmp);
+ auto c = LLVMBuildLoad2(builder, LLVMInt32Type(), tmp, "");
+ auto d = LLVMConstInt(LLVMInt32Type(), 2, 0);
+ auto e = LLVMBuildMul(builder, c, d, "");
+ LLVMBuildRet(builder, e);
+ LLVMDisposeBuilder(builder);
+ //auto buffer = LLVMWriteBitcodeToMemoryBuffer(module);
+ //const auto* start = LLVMGetBufferStart(buffer);
+ //const auto size = LLVMGetBufferSize(buffer);
+ //std::string_view s(start, size);
+ //std::cout<<s<<std::endl;
+ std::cout << LLVMPrintModuleToString(module) << std::endl;
+ auto orcTsCtx = LLVMOrcCreateNewThreadSafeContext();
+
+
+ LLVMOrcLLJITRef lljit;
+ auto orcJitBuilder = LLVMOrcCreateLLJITBuilder();
+ auto orcJitTargetMachineBuilder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(targetMachine);
+ LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(orcJitBuilder, orcJitTargetMachineBuilder);
+ LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(orcJitBuilder, llvm_create_object_layer, NULL);
+ auto err = LLVMOrcCreateLLJIT(&lljit, orcJitBuilder);
+ if (err) {
+ LOG(ERROR) << "error: " << LLVMGetErrorMessage(err);
+ }
+
+ LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit), llvm_log_jit_error, NULL);
 
+ LLVMOrcDefinitionGeneratorRef main_gen, ref_gen;
+ err = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen, LLVMOrcLLJITGetGlobalPrefix(lljit), 0, NULL);
+ if (err) {
+ std::cout << "error" << LLVMGetErrorMessage(err) << std::endl;
+ }
+ LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
+
+ ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(definitionGeneratorFn, NULL);
+ LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
+
+ auto orcTsModule = LLVMOrcCreateNewThreadSafeModule(module, orcTsCtx);
+ LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(lljit);
+ auto rt = LLVMOrcJITDylibCreateResourceTracker(jd);
+
+ err = LLVMOrcLLJITAddLLVMIRModuleWithRT(lljit, rt, orcTsModule);
+ if (err) {
+ LOG(ERROR) << "error: " << LLVMGetErrorMessage(err);
+ }
+ LLVMOrcExecutorAddress addr;
+ err = LLVMOrcLLJITLookup(lljit, &addr, "foobar");
+ if (err) {
+ LOG(ERROR) << "error: " << LLVMGetErrorMessage(err);
+ }
+ auto f = reinterpret_cast<int (*)(int, int)>(reinterpret_cast<uintptr_t>(addr));
+ std::cout << f(10, 100) << std::endl;
 }
 } // namespace test
 int main(int argc, char** argv) {

unittest/test_llvm.cc

@@ -10,6 +10,7 @@
 
 #include <random>
 #include <util/bits_op.hh>
+#include <absl/container/flat_hash_map.h>
 namespace com {
 namespace grakra {
 namespace util {
@@ -104,7 +105,49 @@ TEST_F(TestUtil, testTerityExprReferenceAssignment) {
  std::cout << &b << std::endl;
  std::cout << &c << std::endl;
 }
+#include<list>
+TEST_F(TestUtil, testFlatHashMap) {
+ absl::flat_hash_map<int, int> abc;
+ std::random_device rd;
+ std::mt19937 gen(rd());
+ std::uniform_int_distribution<int8_t> r;
+ std::uniform_int_distribution<size_t> rsize(1, 1000);
+ for (int i = 0; i < 100000; ++i) {
+ const auto size = rsize(gen);
+ for (int k = 0; k < size; ++k) {
+ abc[r(gen)] = r(gen);
+ }
+ for (const auto& [a, b] : abc) {
+ std::cout << "k=" << a << ", v=" << b << std::endl;
+ }
+ }
+}
 
+TEST_F(TestUtil, testVectorResize){
+ std::vector<int> data;
+ for(int i=0;i < 100; i++){
+ data.resize(i*100);
+ std::cout<<"size="<<i*100 <<", cap="<<data.capacity()<<std::endl;
+ }
+}
+
+TEST_F(TestUtil, TestUnsignedIntegerOverflow){
+ uint32_t a = 4294967295;
+ int32_t c = a;
+ int64_t b = c;
+ std::cout<<b<<std::endl;
+}
+
+TEST_F(TestUtil,TestListTransform){
+ std::list<int> l0{1,2,3,4,5};
+ std::list<int> l1;
+ l1.resize(10);
+ std::transform(l0.begin(), l0.end(), l1.begin(), [](int x){return x*2;});
+ std::cout<<"size="<<l1.size()<<std::endl;
+ for (auto it = l1.begin(); it!=l1.end(); ++it) {
+ std::cout<<*it<<std::endl;
+ }
+}
 } // namespace util
 } // namespace grakra
 } // namespace com