Merge branch 'master' of https://github.com/SRI-CSL/whole-program-llvm

Author: ianamason

.travis.yml

@@ -1,21 +1,35 @@
 language: python
+
+sudo: required
+dist: trusty
+
 python:
   - "3.3"
   - "2.7"
   - "2.6"
 
+
 # command to install dependencies
 install:
   - sudo apt-get update
-  - sudo apt-get install llvm-3.4 clang-3.4 libapr1-dev libaprutil1-dev
-  - sudo apt-get install dragonegg llvm-3.0 llvm-gcc
+  # to install wllvm
+  - sudo apt-get install -y python-pip
+  # apache prerequisites
+  - sudo apt-get install  -y libapr1-dev libaprutil1-dev
+  # for the clang build  
+  - sudo apt-get install  -y llvm-3.5 clang-3.5
+  # dragonegg prereqs. dragonegg and llvm-gcc use llvm 3.3
+  - sudo apt-get install  -y llvm-3.3 llvm-gcc-4.7
+  - sudo pip install -e .
   - export WLLVM_HOME=`pwd`
+  - export APACHE_VER=2.4.18
+
 
 # command to run tests
 script:
-  # build apache with clang (i.e. httpd-2.4.12) 
+  # build apache with clang
   - ${WLLVM_HOME}/.travis/apache_clang.sh
-  # build apache with gcc and dragonegg (i.e. httpd-2.4.12) 
+  # build apache with gcc and dragonegg
   - ${WLLVM_HOME}/.travis/apache_dragonegg.sh
 
 

.travis/apache_clang.sh

@@ -3,11 +3,10 @@
 set -e
 
 
-export PATH=/usr/lib/llvm-3.4/bin:${WLLVM_HOME}:${PATH}
+export PATH=/usr/lib/llvm-3.5/bin:${PATH}
 export LLVM_COMPILER=clang
 export WLLVM_OUTPUT=WARNING
 
-APACHE_VER=2.4.18
 wget https://archive.apache.org/dist/httpd/httpd-${APACHE_VER}.tar.gz
 
 tar xfz httpd-${APACHE_VER}.tar.gz

.travis/apache_dragonegg.sh

@@ -4,14 +4,13 @@ set -e
 
 export dragonegg_disable_version_check=true
 
-export PATH=/usr/lib/llvm-3.0/bin:${WLLVM_HOME}:${PATH}
+export PATH=/usr/lib/llvm-3.3/bin:${PATH}
 export LLVM_COMPILER=dragonegg
 export LLVM_GCC_PREFIX=llvm-
-export LLVM_DRAGONEGG_PLUGIN=/usr/lib/gcc/x86_64-linux-gnu/4.6/plugin/dragonegg.so
+export LLVM_DRAGONEGG_PLUGIN=/usr/lib/gcc/x86_64-linux-gnu/4.7/plugin/dragonegg.so
 
 export WLLVM_OUTPUT=WARNING
 
-APACHE_VER=2.4.18
 wget https://archive.apache.org/dist/httpd/httpd-${APACHE_VER}.tar.gz
 
 

LICENSE

@@ -1,3 +1,4 @@
+Copyright (c) 2013-2016 Ian A. Mason, Tristan Ravitch, Bruno Dutertre
 Copyright (c) 2011-2013 Tristan Ravitch
 Copyright (c) 2013 Ben Liblit, Daniel Liew
 

MANIFEST.in

@@ -0,0 +1,2 @@
+include wllvm/dragonegg_as/as
+

Makefile

@@ -0,0 +1,68 @@
+
+all:
+	@echo ''
+	@echo 'Here are the targets:'
+	@echo ''
+	@echo 'To develop          :    "make develop"'
+	@echo 'To test install     :    "make testinstall"'
+	@echo 'To install          :    "make install"'
+	@echo 'To test publish     :    "make testpublish"'
+	@echo 'To publish          :    "make publish"'
+
+	@echo 'To check clang      :    "make check_clang"'
+	@echo ''
+	@echo 'e.g. on linux: LLVM_COMPILER_PATH=/usr/lib/llvm-3.5/bin/ make check_clang'
+	@echo ''
+	@echo 'To check dragonegg  :    "make check_dragonegg"'
+	@echo ''
+	@echo 'e.g. on linux: PATH=/usr/lib/llvm-3.3/bin:... make check_dragonegg'
+	@echo ''
+	@echo 'To turn md 2 html   :    "make zippity"'
+	@echo ''
+	@echo 'then upload the zip file to https://pypi.python.org/pypi'
+	@echo ''
+
+
+
+#local editable install for developing
+develop: 
+	pip install -e .
+
+
+dist: clean
+	python setup.py bdist_wheel
+
+# If you need to push this project again,
+# INCREASE the version number in wllvm/version.py,
+# otherwise the server will give you an error. 
+
+testpublish: dist
+	python setup.py register -r https://testpypi.python.org/pypi
+	python setup.py sdist upload -r https://testpypi.python.org/pypi
+
+testinstall:
+	pip install -i https://testpypi.python.org/pypi wllvm
+
+publish: dist
+	python setup.py register -r https://pypi.python.org/pypi
+	python setup.py sdist upload -r https://pypi.python.org/pypi
+
+install:
+	pip install 
+
+check_clang:
+	cd test; python -m unittest -v test_base_driver test_clang_driver
+
+check_dragonegg:
+	cd test; python -m unittest -v test_base_driver test_dragonegg_driver
+
+
+zippity:
+	rm -rf doczip*; mkdir doczip;
+	cat README.md | pandoc -f markdown_github > doczip/index.html
+	zip -r -j doczip.zip doczip
+
+clean:
+	rm -f  wllvm/*.pyc wllvm/*~
+
+

README.md

@@ -31,11 +31,29 @@ of a build process requires that.
 
 WLLVM works with either clang or the gcc dragonegg plugin.
 
+Installation
+============
+
+As of August 2016 WLLVM is now a pip package. You can just do:
+
+    pip install wllvm
+
+
+If you want to develop or use the development version:
+
+    git clone https://github.com/SRI-CSL/whole-program-llvm
+    cd whole-program-llvm
+    pip install -e .
+
+
 Usage
 =====
 
 WLLVM includes two python executables: `wllvm` for compiling C code
-and `wllvm++` for C++, and an auxiliary tool `extract-bc`.
+and `wllvm++` for compiling C++, and an auxiliary tool `extract-bc` for
+extracting the bitcode from a build product (object file, executable, library
+or archive). There is also a sanity checker, `wllvm-sanity-checker` for detecting
+configuration oversights.
 
 Three environment variables must be set to use these wrappers:
 
@@ -74,6 +92,9 @@ In addition to the above environment variables the following can be optionally u
    bitcode files.
 
 
+
+
+
 Building a bitcode module with clang
 ====================================
 
@@ -84,11 +105,18 @@ Building a bitcode module with clang
     CC=wllvm ./configure
     make
 
-    # Produces pkg-config.bc
+This should produce the executable `pkg-config`. To extract the bitcode:
+
     extract-bc pkg-config
 
-A gentler set of instructions on building apache can be found
-[here.](https://github.com/SRI-CSL/whole-program-llvm/blob/master/tutorial.md)
+which will produce the bitcode module `pkg-config.bc`.
+
+
+Tutorials
+=========
+
+A gentler set of instructions on building apache in a vagrant Ubuntu 14.04 can be found
+[here,](https://github.com/SRI-CSL/whole-program-llvm/blob/master/doc/tutorial.md) and for Ubuntu 16.04 [here.](https://github.com/SRI-CSL/whole-program-llvm/blob/master/doc/tutorial-ubuntu-16.04.md)
 
 Building a bitcode module with dragonegg
 ========================================
@@ -102,9 +130,12 @@ Building a bitcode module with dragonegg
     CC=wllvm ./configure
     make
 
-    # Produces pkg-config.bc
+Again, this should produce the executable `pkg-config`. To extract the bitcode:
+
     extract-bc pkg-config
 
+which will produce the bitcode module `pkg-config.bc`.
+
 
 Building bitcode archive
 ========================
@@ -119,17 +150,30 @@ Building bitcode archive
     # Produces src/LinearMath/libLinearMath.bca
     extract-bc src/LinearMath/libLinearMath.a
 
+Note that by default extracting bitcode from an archive produces
+an archive of bitcode. You can also extract the bitcode directly into a module.
+
+    extract-bc -b src/LinearMath/libLinearMath.a
+
+produces `src/LinearMath/libLinearMath.a.bc`.
+
+
 
 Building an Operating System
 ============================
 
-To see how to build freeBSD 10.0 from scratch check out this 
-[guide.](../master/README-freeBSD.md)
+To see how to build freeBSD 10.0 from scratch check out this
+[guide.](https://github.com/SRI-CSL/whole-program-llvm/blob/master/doc/tutorial-freeBSD.md)
 
 
 Configuring without building bitcode
 ================================
 
+Sometimes it is necessary to disable the production of bitcode.
+Typically this is during configuration, where the production
+of unexpected files can confuse the configure script. For this
+we have a flag `WLLVM_CONFIGURE_ONLY` which can be used as
+follows:
 
     WLLVM_CONFIGURE_ONLY=1 CC=wllvm ./configure
     CC=wllvm make
@@ -149,9 +193,11 @@ Building a bitcode archive then extracting the bitcode
     extract-bc libjansson.a
     llvm-ar x libjansson.bca
     ls -la
-   
+
 
 
+
+
 Debugging
 =========
 
@@ -183,4 +229,4 @@ it might point out what is wrong.
 License
 =======
 
-WLLVM is released under the MIT license. See the file `LICENSE` for details.
+WLLVM is released under the MIT license. See the file `LICENSE` for [details.](https://github.com/SRI-CSL/whole-program-llvm/blob/master/LICENSE)

README.rst

@@ -0,0 +1,80 @@
+Introduction to WLLVM
+=====================
+
+This project, WLLVM, provides tools for building whole-program (or
+whole-library) LLVM bitcode files from an unmodified C or C++
+source package. It currently runs on `*nix` platforms such as Linux,
+FreeBSD, and Mac OS X.
+
+WLLVM provides python-based compiler wrappers that work in two
+steps. The wrappers first invoke the compiler as normal. Then, for
+each object file, they call a bitcode compiler to produce LLVM
+bitcode. The wrappers also store the location of the generated bitcode
+file in a dedicated section of the object file.  When object files are
+linked together, the contents of the dedicated sections are
+concatenated (so we don't lose the locations of any of the constituent
+bitcode files). After the build completes, one can use an WLLVM
+utility to read the contents of the dedicated section and link all of
+the bitcode into a single whole-program bitcode file. This utility
+works for both executable and native libraries.
+
+This two-phase build process is necessary to be a drop-in replacement
+for ``gcc`` or ``g++`` in any build system.  Using the LTO framework in gcc
+and the gold linker plugin works in many cases, but fails in the
+presence of static libraries in builds.  WLLVM's approach has the
+distinct advantage of generating working binaries, in case some part
+of a build process requires that.
+
+WLLVM works with either ``clang`` or the ``gcc dragonegg`` plugin.
+
+
+Usage
+-----
+
+WLLVM includes two python executables: ``wllvm`` for compiling C code
+and ``wllvm++`` for C++, and an auxiliary tool ``extract-bc``.
+
+Three environment variables must be set to use these wrappers:
+
+ * ``LLVM_COMPILER`` should be set to either ``dragonegg`` or ``clang``.
+ * ``LLVM_GCC_PREFIX`` should be set to the prefix for the version of gcc that should
+   be used with dragonegg.  This can be empty if there is no prefix.  This variable is
+   not used if ``$LLVM_COMPILER == clang``.
+ * ``LLVM_DRAGONEGG_PLUGIN`` should be the full path to the dragonegg plugin.  This
+   variable is not used if ``$LLVM_COMPILER == clang``.
+
+Once the environment is set up, just use ``wllvm`` and ``wllvm++`` as your C
+and C++ compilers, respectively.
+
+
+In addition to the above environment variables the following can be optionally used:
+
+ * ``LLVM_CC_NAME`` can be set if your clang compiler is not called ``clang`` but
+   something like ``clang-3.7``. Similarly ``LLVM_CXX_NAME`` can be used to describe
+   what the C++ compiler is called. Note that in these sorts of cases, the environment
+   variable ``LLVM_COMPILER`` should still be set to ``clang`` not ``clang-3.7`` etc.
+   We also pay attention to the environment variables ``LLVM_LINK_NAME`` and ``LLVM_AR_NAME`` in an
+   analagous way,  since they too get adorned with suffixes in various Linux distributions.
+
+ * ``LLVM_COMPILER_PATH`` can be set to the absolute path to the folder that
+   contains the compiler and other LLVM tools such as ``llvm-link`` to be used.
+   This prevents searching for the compiler in your PATH environment variable.
+   This can be useful if you have different versions of clang on your system
+   and you want to easily switch compilers without tinkering with your PATH
+   variable.
+   Example ``LLVM_COMPILER_PATH=/home/user/llvm_and_clang/Debug+Asserts/bin``.
+
+ * ``WLLVM_CONFIGURE_ONLY`` can be set to anything. If it is set, ``wllvm``
+   and ``wllvm++`` behave like a normal C or C++ compiler. They do not
+   produce bitcode.  Setting ``WLLVM_CONFIGURE_ONLY`` may prevent
+   configuration errors caused by the unexpected production of hidden
+   bitcode files.
+
+
+Documentation
+-------------
+
+More detailed documentation as well as some tutorials can be found
+here:
+
+https://github.com/SRI-CSL/whole-program-llvm