Friday, January 6, 2017

Using Python with ns-3

Though ns-3 is written in C++, it has an Python binding. The code sample below shows how to use Python to create an network experiment.

  • It creates a network of three nodes. Two of them are connected via point to point link and two of them are connected via Ethernet.

        # Topology
        #         10.1.1.0
        #    n0 -------------- n1   n2
        #       point-to-point  |    |
        #                       ======
        #                     LAN 10.1.2.0
  • It setup a Tcp socket client on node n2 which keep sending packets to a Tcp socket server on node n0.

  • To simulate packet drops, a error model was used to generate 1 packet drop per 100 packets.
  • The data transfer is captured via Pcap.

Tip
The ns.core module is created in a subtle way in ns-3 Python binding that most of the Python IDEs cannot get the intellisense from the module. That is because all the classes/functions in ns.core are imported dynamically via code (See [ns_root]/src/core/bindings/core.py).

The only way so far I found to work around it is remove the core.py file and rebuild. After that a core.so module will be generated instead of a _core.so. IDEs are happy with core.so.



Here is the code:

import sys
from ns3 import *


def create_nodes(n):
    return (Node() for _ in range(n))


def node_container(*nodes):
    c = NodeContainer()
    for n in nodes:
        c.Add(n)
    return c


def connect_nodes_with_p2p_channel(nodes, rate, delay, pcap_name):
    helper = PointToPointHelper()
    helper.SetDeviceAttribute("DataRate", StringValue(rate))
    helper.SetChannelAttribute("Delay", StringValue(delay))
    devices = helper.Install(nodes)
    if pcap_name is not None:
        helper.EnablePcapAll(pcap_name)
    return devices


def connect_nodes_with_csma_channel(nodes, rate, delay, pcap_name):
    helper = CsmaHelper()
    helper.SetChannelAttribute("DataRate", StringValue(rate))
    helper.SetChannelAttribute("Delay", StringValue(delay))
    devices = helper.Install(nodes)
    if pcap_name is not None:
        helper.EnablePcapAll(pcap_name)
    return devices


def install_internet_stack(nodes):
    stack = InternetStackHelper()
    stack.Install(nodes)


def set_receive_error(device, error_packet_seq_ids):
    of = ObjectFactory()
    of.SetTypeId('ns3::ReceiveListErrorModel')
    em = of.Create()
    em.SetList(error_packet_seq_ids)
    device.SetAttribute("ReceiveErrorModel", PointerValue(em))
    return em


def create_interfaces(devices, ip_base, mask):
    address = Ipv4AddressHelper()
    address.SetBase(Ipv4Address(ip_base), Ipv4Mask(mask))
    interfaces = address.Assign(devices)
    return interfaces


def get_protocol_type_id(protocol):
    if protocol == 'tcp':
        return 'ns3::TcpSocketFactory'
    elif protocol == 'udp':
        return 'ns3::UdpSocketFactory'
    else:
        return None


def install_packet_sink_app(node, protocol, port):
    protocol_type_id = get_protocol_type_id(protocol)
    helper = PacketSinkHelper(protocol_type_id, Address(InetSocketAddress(Ipv4Address.GetAny(), port)))
    app = helper.Install(node)
    return app


def install_on_off_app(node, protocol, server_address, server_port, rate=None, packet_size=None, on_time=None, off_time=None):
    protocol_type_id = get_protocol_type_id(protocol)
    helper = OnOffHelper(protocol_type_id, Address(InetSocketAddress(server_address, server_port)))
    if rate is not None:
        helper.SetAttribute("DataRate", DataRateValue(DataRate(rate)))
    if packet_size is not None:
        helper.SetAttribute("PacketSize", UintegerValue(packet_size))
    if on_time is not None:
        helper.SetAttribute("OnTime", get_constant_random_variable(on_time))
    if off_time is not None:
        helper.SetAttribute("OffTime", get_constant_random_variable(off_time))

    app = helper.Install(node)
    return app


def get_constant_random_variable(constant):
    # return StringValue('ns3::ConstantRandomVariable[Constant=%d]' % constant)
    v = ConstantRandomVariable()
    v.SetAttribute('Constant', StringValue(str(constant)))
    return PointerValue(v)


def main():
    cmd = CommandLine()
    cmd.Parse(sys.argv)

    # Topology
    #         10.1.1.0
    #    n0 -------------- n1   n2
    #       point-to-point  |    |
    #                       ======
    #                     LAN 10.1.2.0

    n0, n1, n2 = create_nodes(3)

    p2p_nodes = node_container(n0, n1)
    p2p_devices = connect_nodes_with_p2p_channel(p2p_nodes, '2Mbps', '2ms', 'p2p')

    csma_nodes = node_container(n1, n2)
    csma_devices = connect_nodes_with_csma_channel(csma_nodes, '10Mbps', '1ms', 'csma')

    install_internet_stack(n0)
    install_internet_stack(n1)
    install_internet_stack(n2)

    set_receive_error(p2p_devices.Get(0), [100 * i for i in range(10)])

    p2p_interfaces = create_interfaces(p2p_devices, '10.1.1.0', '255.255.255.0')
    csma_interfaces = create_interfaces(csma_devices, '10.1.2.0', '255.255.255.0')

    server_address = p2p_interfaces.GetAddress(0)
    server_port = 8888
    server_app = install_packet_sink_app(n0, 'tcp', server_port)
    server_app.Start(Seconds(1.0))
    server_app.Stop(Seconds(10.0))

    client_address = csma_interfaces.GetAddress(1)
    on_off_app = install_on_off_app(n3, 'tcp', server_address, server_port, rate='2Mbps', on_time=1, off_time=0)
    on_off_app.Start(Seconds(2.0))
    on_off_app.Stop(Seconds(10.0))

    Ipv4GlobalRoutingHelper.PopulateRoutingTables()

    Simulator.Stop(Seconds(20.0))
    Simulator.Run()
    Simulator.Destroy()


def log_cwnd(oldval, newval):
    print('cwnd old: %d, new: %d' % (oldval, newval))


def start_flow(socket, address, port):
    def write_until_buffer_full(socket, available):
        print(available)
        n = 10000
        while socket.GetTxAvailable() > 0 and n > 0:
            to_write = min(4 * 1024 * 1024, socket.GetTxAvailable())
            sent = socket.Send(Packet(to_write), 0)
            if sent < 0:
                return
            n -= 1

    socket.Connect(InetSocketAddress(address, port))
    socket.SetSendCallback(write_until_buffer_full)
    write_until_buffer_full(socket, socket.GetTxAvailable())


if __name__ == '__main__':
    main()

Visualize ns-3 network using PyViz

The steps below can be use to enable the PyViz visualizer for ns-3 python binding. For C++ code, you could follow the steps in the referenced post.

  1. Install prerequisites

    sudo apt-get install python-dev python-pygraphviz python-kiwi python-pygoocanvas python-gnome2 python-gnomedesktop python-rsvg
  2. Change your python code

    import ns.visualizer
    [...]
    cmd = ns.core.CommandLine()
    cmd.Parse(sys.argv)
    [...]
    ns.core.Simulator.Run()
  3. Run your script with command line argument

    your_script.py --SimulatorImplementationType=ns3::VisualSimulatorImpl

    or

    ./waf --pyrun your_script.py --vis



Reference: PyViz

Thursday, January 5, 2017

Hello World from Network Simulator

ns-3 is a fabulous network simulator. It allows you to setup a custom network environment and do experiments with it. The reason I want to play with it is to learn what TCP Tahoe differs from TCP Reno in fact (instead of in theory).

Here is a list of steps to get started with ns-3.

  1. Get a Ubuntu VM

  2. Install prerequisits
    a. You could follow this document for details
    b. Basically, run the following command

    sudo apt-get install gcc g++ python gcc g++ python python-dev qt4-dev-tools libqt4-dev mercurial bzr cmake libc6-dev g++-multilib gdb valgrind gsl-bin libgsl2 flex bison libfl-dev tcpdump sqlite sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk2.0-0 libgtk2.0-dev vtun lxc uncrustify doxygen graphviz imagemagick texlive texlive-extra-utils texlive-latex-extra texlive-font-utils texlive-lang-portuguese dvipng python-sphinx dia python-pygraphviz python-kiwi python-pygoocanvas libgoocanvas-dev ipython libboost-signals-dev libboost-filesystem-dev openmpi-bin openmpi-common openmpi-doc libopenmpi-dev
  3. Download ns-3 sources

    cd
    mkdir workspace
    cd workspace
    wget http://www.nsnam.org/release/ns-allinone-3.26.tar.bz2
    tar xjf ns-allinone-3.26.tar.bz2
  4. Build ns-3

    cd ns-allinone-3.26
    ./build.py --enable-examples --enable-tests
  5. Create a hello world program

    cd ns-3.25/
    cp examples/tutorial/first.cc scratch/

    You could view the code in the end of the post.

  6. Build and run the program

    ./waf
    ./waf --run scratch/first

    The output of the program will be something like the following.

    At time 2s client sent 1024 bytes to 10.1.1.2 port 9
    At time 2.00369s server received 1024 bytes from 10.1.1.1 port 49153
    At time 2.00369s server sent 1024 bytes to 10.1.1.1 port 49153
    At time 2.00737s client received 1024 bytes from 10.1.1.2 port 9
  7. You can also use the Python binding to write the program. However it is still a work in progress and there are some limitations.

Sample Code

/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation;
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

#include "ns3/core-module.h"
#include "ns3/network-module.h"
#include "ns3/internet-module.h"
#include "ns3/point-to-point-module.h"
#include "ns3/applications-module.h"

using namespace ns3;

NS_LOG_COMPONENT_DEFINE ("FirstScriptExample");

int
main (int argc, char *argv[])
{
  Time::SetResolution (Time::NS);
  LogComponentEnable ("UdpEchoClientApplication", LOG_LEVEL_INFO);
  LogComponentEnable ("UdpEchoServerApplication", LOG_LEVEL_INFO);

  NodeContainer nodes;
  nodes.Create (2);

  PointToPointHelper pointToPoint;
  pointToPoint.SetDeviceAttribute ("DataRate", StringValue ("5Mbps"));
  pointToPoint.SetChannelAttribute ("Delay", StringValue ("2ms"));

  NetDeviceContainer devices;
  devices = pointToPoint.Install (nodes);

  InternetStackHelper stack;
  stack.Install (nodes);

  Ipv4AddressHelper address;
  address.SetBase ("10.1.1.0", "255.255.255.0");

  Ipv4InterfaceContainer interfaces = address.Assign (devices);

  UdpEchoServerHelper echoServer (9);

  ApplicationContainer serverApps = echoServer.Install (nodes.Get (1));
  serverApps.Start (Seconds (1.0));
  serverApps.Stop (Seconds (10.0));

  UdpEchoClientHelper echoClient (interfaces.GetAddress (1), 9);
  echoClient.SetAttribute ("MaxPackets", UintegerValue (1));
  echoClient.SetAttribute ("Interval", TimeValue (Seconds (1.0)));
  echoClient.SetAttribute ("PacketSize", UintegerValue (1024));

  ApplicationContainer clientApps = echoClient.Install (nodes.Get (0));
  clientApps.Start (Seconds (2.0));
  clientApps.Stop (Seconds (10.0));

  Simulator::Run ();
  Simulator::Destroy ();
  return 0;
}

Shrink a Ubuntu Guest VDI File

When using a Ubuntu virtual box guest, sometimes we want to shrink the dynamic expanding VDI file to release the unused disk space. Here is the steps to archive that.

  1. Install ZeroFree

    sudo apt-get install zerofree
  2. Reboot into recovery mode
    a. sudo shutdown -r now
    b. Holding the left Shift key while rebooting
    c. Select Advanced and then Recovery mode and finally Drop to root shell prompt option.

  3. Remount the root partition

    mount -n -o remount,ro -t ext4 /dev/sda1 /
  4. Run ZeroFree

    sudo zerofree -v /dev/sda1
  5. Shutdown the VM

    shutdown -h now
  6. Shrink the VDI file is host OS

    SET PATH=%PATH%;"c:\Program Files\Oracle\VirtualBox
    VBoxManage modifyhd Ubuntu.vdi -compact


Reference: How to shrink a dynamically-expanding guest virtualbox image

Monday, October 6, 2014

Install OEM Windows in VirtualBox

The following steps can be followed to install OEM windows onto a virtual box.

  • Create new virtual machine
  • Edit the .vdm file of the virtual machine and add the following content under the element.
<ExtraDataItem name="VBoxInternal/Devices/pcbios/0/Config/DmiBIOSVendor" value="LENOVO"/>
<ExtraDataItem name="VBoxInternal/Devices/pcbios/0/Config/DmiSystemVendor" value="LENOVO"/>
  • Install OEM Windows onto the virtual machine.

In the example, the OEM is for IBM Corporation.

Friday, April 25, 2014

Build XulRunner with Visual C++ 2013

1. Prerequisites

1.1 MozillaBuild

1.2 Visual Studio Express 2013 for Windows Desktop

1.3 [June 2010 DirectX SDK] http://www.microsoft.com/download/en/details.aspx?displaylang=en&id=6812

  • If you met with S1023 error whiling installing the DirectX SDK, please uninstall all Microsoft Visual C++ 2010 x86/x64 Redistributable packages (See this for details).

2. Source code

  • Download XulRunner source code from here
  • Extract to c:\build\mozilla
  • Apply the patch for vc12
  • Add #include <algorith> to C:\build\mozilla\toolkit\components\protobuf\google\protobuf\wire_format_lite_inl.h

3. Configure (x86)

Create a .mozconfig file in c:\build\mozilla folder. Enter the following content:

mk_add_options MOZ_CO_PROJECT=xulrunner
mk_add_options MOZ_OBJDIR=c:/build/mozilla/obj-x86
ac_add_options --enable-application=xulrunner

ac_add_options --disable-debug
ac_add_options --disable-debug-symbols
ac_add_options --disable-javaxpcom
ac_add_options --disable-tests
ac_add_options --disable-ipdl-tests
ac_add_options --disable-activex
ac_add_options --disable-activex-scripting
ac_add_options --disable-installer
ac_add_options --disable-crashreporter
ac_add_options --disable-updater
ac_add_options --disable-update-channel
ac_add_options --disable-update-packaging
ac_add_options --disable-maintenance-service
ac_add_options --disable-accessibility
ac_add_options --disable-logging
ac_add_options --disable-services-healthreport
ac_add_options --disable-telemetry-reporting 
ac_add_options --disable-parental-controls 
ac_add_options --disable-windows-mobile-components 
ac_add_options --disable-necko-wifi
ac_add_options --disable-pdfjs
ac_add_options --disable-accessibility
ac_add_options --disable-gamepad

ac_add_options --with-windows-version=601

4.Build (x86)

Start /start-shell-msvc2013.bat, and run the commands below.

cd /c/build/mozilla
python build/pymake/make.py  -f client.mk build
python build/pymake/make.py  -f client.mk sdk

Once the build completed, the sdk archive can be found in C:\build\mozilla\obj-x86\dist\sdk.

For more detailed documentation. See Mozilla’s Official Build Instructions

Thursday, April 24, 2014

Download a Folder from Github

Here is way to download a single folder from a Github repository without fully clone the repo.

For example, if you want to download https://github.com/jack/foo/tree/master/bar folder, you could use the command below.

svn checkout https://github.com/jack/foo/trunk/bar

Note that the tree/master is replaced with trunk.

Monday, April 14, 2014

Delete System Files On Windows 7

On Windows 7, system file are protected from being deleted. The following commands can be used to delete system files.

set THE_FILE_TO_DELETE=C:\Windows\IME\IMESC5\DICTS\PINTLGT.IMD
takeown /f %THE_FILE_TO_DELETE%
cacls %THE_FILE_TO_DELETE% /G YourUserName:F
del %THE_FILE_TO_DELETE%

Create Windows PE 5.1 Image

Listed below are the instructsions to create a Windows PE 5.1 x86 disk. To build an amd64 version, replace all ‘x86’ to ‘amd64’ in the commands below.

Prerequisites

Steps

  • Start “Deployment and Imaging Tools Environment” as administrator

  • Copy a basic PE image.

copype x86 C:\WinPE_x86
  • Mount the PE image to local directory
Dism /Mount-Image /ImageFile:"C:\WinPE_x86\media\sources\boot.wim" /index:1 /MountDir:"C:\WinPE_x86\mount"
  • (Optional) Add packages. For example, the following command adds .Net Framework to the PE
Dism /Add-Package /Image:"C:\WinPE_x86\mount" /PackagePath:"C:\Program Files (x86)\Windows Kits\8.1\Assessment and Deployment Kit\Windows Preinstallation Environment\x86\WinPE_OCs\WinPE-NetFx.cab"
  • (Optional) Verify packages using the following command (optional)
Dism /Get-Packages /Image:"C:\WinPE_x86\mount"
  • (Optional) Add thridparty programs. For example, the following command adds 7Zip to the PE
copy "C:\Program Files (x86)\7-Zip\7z*.*" C:\WinPE_x86\mount\Windows\System32
  • (Optional) Change PE boot drive letter. For example the following command change the PE’s boot drive to V:
Dism /image:C\WinPE_x86\mount /Set-TargetPath:V:\
  • Unmount the image and commit changes
Dism /Unmount-Image /MountDir:"C:\WinPE_x86\mount" /commit
  • Deploy the PE image to an ISO
MakeWinPEMedia /ISO C:\WinPE_x86 C:\WinPE_x86\WinPE_x86.iso
  • Deploy the PE image to an USB drive (e.g. U:)
MakeWinPEMedia /UFD C:\WinPE_x86 U:

Saturday, March 29, 2014

Windows Batch Scripting Tips

Find files recursively

for /r %%f in (*.abc) do echo %%f

or

for /f "delims=|" %%f in ('dir /B /S *.abc') do echo %%f

Block of statements

(The parentheses must be used exactly as shown)

if %VAR% == 0 (
    echo "statement 1"
    echo "statement 2"
)

Thursday, March 20, 2014

Setup Schroot Environment On Ubuntu

Here is my note about how to setup Ubuntu Lucid schroot environment on a Ubuntu box. Please refer to Ubuntu Document for details.

Software Prerequisits

sudo apt-get install debootstrap
sudo apt-get install schroot

Create a config file for Ubuntu Lucid (x86)

sudo vi /etc/schroot/chroot.d/lucid_i386.conf

Fill in the following content. Please make sure change the user bob to your actual username.

[lucid_i386]
description=Ubuntu 10.04 for i386
directory=/srv/chroot/lucid_i386
personality=linux32
type=directory
root-users=bob
users=bob

Get environment files

sudo mkdir -p /srv/chroot/lucid_i386
sudo debootstrap --arch=i386 lucid /srv/chroot/lucid_i386/ http://archive.ubuntu.com/ubuntu/

Done. Verify environment works

schroot -l
schroot -c lucid_i386 -u root
lsb_release -a

Here is the steps for Ubuntu Lucid (x64)

sudo vi /etc/schroot/chroot.d/lucid_amd64.conf
[lucid_amd64]
description=Ubuntu 10.04 for amd64
directory=/srv/chroot/lucid_amd64
#personality=linux32
type=directory
root-users=bob
users=bob
sudo mkdir -p /srv/chroot/lucid_amd64
sudo debootstrap --arch=amd64 lucid /srv/chroot/lucid_amd64/ http://archive.ubuntu.com/ubuntu/

For Debian Squeeze (x64)

sudo vi /etc/schroot/chroot.d/squeeze_amd64.conf
[squeeze_amd64]
description=Ubuntu 10.04 for amd64
directory=/srv/chroot/squeeze_amd64
#personality=linux32
type=directory
root-users=bob
users=bob
sudo mkdir -p /srv/chroot/squeeze_amd64
sudo debootstrap --arch=amd64 squeeze /srv/chroot/squeeze_amd64/ http://ftp.debian.org/debian/

Saturday, March 15, 2014

Setup Debian in a Virtualbox

The following tips can make it convenient to use debian in a virtual box, but please keep in mind that they are not good practices for a production server.

Disable Sudo Password Requirement

On ubuntu/debian, you could disable sudo password prompt by following the steps below.

sudo visudo

Change

%sudo ALL=(ALL) ALL

to

%sudo ALL=(ALL) NOPASSWD: ALL

Auto login

The following steps can let you log in automatically to a console when boots.

sudo vi /etc/inittab

Change the line

1:2345:respawn:/sbin/getty 38400 tty1

to

1:2345:respawn:/bin/login -f user </dev/tty1 >/dev/tty1 2>&1

Disable Grub Screen

The following steps can disable grub splash screen when boots

sudo vi /etc/default/grub

Change the value of GRUB_TIMEOUT to 0

And then run the following command to update grub config.

sudo update-grub

Friday, February 21, 2014

Targeting Windows XP with Visual C++ 2013

Build apps using Visual C++ 2013 that target Windows XP, special steps are needed.

Environment Variables to Set

The following codes is quoted from this blog.

set INCLUDE=%ProgramFiles(x86)%\Microsoft SDKs\Windows\7.1A\Include;%INCLUDE%
set PATH=%ProgramFiles(x86)%\Microsoft SDKs\Windows\7.1A\Bin;%PATH%
set LIB=%ProgramFiles(x86)%\Microsoft SDKs\Windows\7.1A\Lib;%LIB%
set CL=/D_USING_V110_SDK71_;%CL%

For building x64 version, change LIB to this one.

set LIB=%ProgramFiles(x86)%\Microsoft SDKs\Windows\7.1A\Lib\x64;%LIB%

For x86 console/windows applications, set /SUBSYSTEM accordingly:

:: for console application
set LINK=/SUBSYSTEM:CONSOLE,5.01 %LINK%
:: for gui application
set LINK=/SUBSYSTEM:WINDOWS,5.01 %LINK%

For x64 console/windows applications:

:: for console application
set LINK=/SUBSYSTEM:CONSOLE,5.02 %LINK%
:: for gui application
set LINK=/SUBSYSTEM:WINDOWS,5.02 %LINK%

Work with CMake

In CMake there is a WIN32 parameter for add_executable command. It works like this:

add_executable(MyExe WIN32 main.cpp) # this exe will link with /SUBSYSTEM:WINDOWS
add_executable(AnotherExe main.cpp)  # this one will link with /SUBSYSTEM:CONSOLE

When we not target Windows XP, it works correctly. But when we target Windows XP, obviously the link flags are not correct. To fix that, we need add a file to override make rules in CMake. CMAKE_USER_MAKE_RULES_OVERRIDE_CXX is for that purpose.

In your project file, add the following code.

cmake_minimum_required(VERSION 2.8)
set(CMAKE_USER_MAKE_RULES_OVERRIDE_CXX your_path/overrides.cmake)

# please make sure the variable is set before the `project` command.
# project(MyApp) 

Then in overrides.cmake, add the following codes.

if(WIN32)
    # if target winxp
    if(TARGETING_XP_64)
        SET(CMAKE_CREATE_WIN32_EXE /SUBSYSTEM:WINDOWS,5.02)
        SET(CMAKE_CREATE_CONSOLE_EXE /SUBSYSTEM:CONSOLE,5.02)
    elif(TARGETING_XP)
        SET(CMAKE_CREATE_WIN32_EXE /SUBSYSTEM:WINDOWS,5.01)
        SET(CMAKE_CREATE_CONSOLE_EXE /SUBSYSTEM:CONSOLE,5.01)
    endif()
endif()

Now, to build targetting Windows XP, you could run cmake like the folowing.

cmake -G "YourGenerator" -DTARGETING_XP=On path_to_your_project.cmake

And the following for XP x64.

cmake -G "YourGenerator" -DTARGETING_XP_64=On path_to_your_project.cmake

Tuesday, February 18, 2014

Core Dump on Ubuntu

Listed below are just several notes about core dump on Ubuntu.

The location of core dump files

The following command can print out where will the core dump files be placed.

cat /proc/sys/kernel/core_pattern

If the output start with a | character, the kernel then will write the core dump to the stdin of the command after the | character.

Core dump file size

The following command can remove core dump file size limit.

ulimit -c unlimited

Apport

On ubuntu, core dump files are forwarded to Apport by default. So if you can’t find the core file in your current directory, you may want to try /var/crash. Probably you will see crash report files there.

The command apport-unpack can be used to extract the core files out of a crash report files.

Friday, February 14, 2014

RPATH on Windows

On Linux, we can specify shared library search paths for our executable/libraries using RPATH. On Windows, however, there is no such easy way to do that. A possible solution is to use /DELAYLOAD and SetDllDirectory.

Ways to Load a DLL

Basically speaking, there are three ways to load a DLL on windows.
- Implicitly loading. In this way, we link our application against the .lib file of the DLL. The DLL will be loaded once the application was invoked.
- Explicitly loading. That is the way we call LoadLibrary Win32 API to load a DLL. The search path can be changed by calling SetDllDirectory function.
- Delayed loading. Similar to Implicit loading, but the DLL won’t be loaded until once of functions in it got called.

Delay loading

/DELAYLOAD is a linker option of MSVC compiler (version 6 or later) that can tell the linker which DLL we want to delay load. So, to simulate the functionality of RPATH, we can specify the DLL should be delay loaded via /DELAYLOAD at link time, and call SetDllDirectory function to set the search path. After that, there are three ways to force load the library.

  • We can simply call a function that is exported in that library to force loading the library.
  • Call LoadLibrary to explicitly load the library.
  • Call __HrLoadAllImportsForDll to force load the library.

Listed below is an example to use the above method to delay load a DLL. Please note that to make the code work, delayimp.lib should be linked against.

#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <delayimp.h>
#include <string>

bool loadTheLibraryNow(const char* dir) {
    const char* baseDllName = "library-to-delay-load.dll";

    if (!dir)
        return false;

    // backup current dll directory
    char previousDllDirectory[MAX_PATH] = { '\0' };
    if (!::GetDllDirectoryA(MAX_PATH, previousDllDirectory)) {
        previousDllDirectory[0] = '\0';
    }

    // set new current dll directory to
    if (!::SetDllDirectoryA(dir))
        return false;

    // force load the dll
    bool result = SUCCEEDED(__HrLoadAllImportsForDll(baseDllName));

    // restore the dll directory to old value
    ::SetDllDirectoryA(previousDllDirectory);

    return result;
}

Please See here for more details.

Build Qt5 Statically with Visual C++ 2013

1. ICU

1.1 Prerequisites

  • Download Cygwin installer from here
  • Install Cygwin and make sure make package is selected

1.2 Download ICU

  • Download ICU from here
  • Extract the zip file to C:\build\icu and make sure folder C:\build\icu\source exists
  • Open c:\build\icu\source\runConfigureICU
  • Replace all /MD to /MT (of course that will also replace /MDd to /MTd)

1.4 Build ICU

x86 version

:: Start a command prompt and setup x86 build env

cd c:\build\icu\source
set PATH=C:\cygwin\bin;%PATH%
dos2unix *
dos2unix -f configure
"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" x86

:: Build the x86 static release version

mkdir c:\build\icu\build\x86-static-release && cd c:\build\icu\build\x86-static-release
bash ../../source/runConfigureICU Cygwin/MSVC --prefix=/cygdrive/c/build/icu/dist/x86-static-release --enable-static --disable-shared
make -j8 && make install

:: Build the x86 static debug version

mkdir c:\build\icu\build\x86-static-debug && cd c:\build\icu\build\x86-static-debug
bash ../../source/runConfigureICU --enable-debug --disable-release Cygwin/MSVC --prefix=/cygdrive/c/build/icu/dist/x86-static-debug --enable-static --disable-shared 
make && make install

::

x64 version

:: Start a command prompt and setup x64 build env

cd c:\build\icu\source
set PATH=C:\cygwin\bin;%PATH%
"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" x86_amd64

:: Build the x64 static release version

mkdir c:\build\icu\build\x64-static-release && cd c:\build\icu\build\x64-static-release
bash ../../source/runConfigureICU Cygwin/MSVC --prefix=/cygdrive/c/build/icu/dist/x64-static-release --enable-static --disable-shared
make -j8 && make install

:: Build the x64 static debug version

mkdir c:\build\icu\build\x64-static-debug && cd c:\build\icu\build\x64-static-debug
bash ../../source/runConfigureICU --enable-debug --disable-release Cygwin/MSVC --prefix=/cygdrive/c/build/icu/dist/x64-static-debug --enable-static --disable-shared
make && make install

::

2. Qt

2.1 Prerequisites

2.2 Change files

  • Open c:\build\qt-everywhere-opensource-src-5.2.1\qtbase\mkspecs\win32-msvc2013\qmake.conf
    • Remove embed_manifest_dll embed_manifest_exe from the line CONFIG += line
    • Make the following change
QMAKE_CFLAGS_RELEASE    = -O2 -MD
QMAKE_CFLAGS_RELEASE_WITH_DEBUGINFO += -O2 -MD -Zi
QMAKE_CFLAGS_DEBUG      = -Zi -MDd

to

QMAKE_CFLAGS_RELEASE    = -O2 -MT
QMAKE_CFLAGS_RELEASE_WITH_DEBUGINFO += -O2 -MT -Zi -Fd$(DESTDIR)$(QMAKE_TARGET).pdb
QMAKE_CFLAGS_DEBUG      = -Zi -MTd -Fd$(DESTDIR)$(QMAKE_TARGET).pdb

2.3 Build Qt Base

x86-release version

  • Extract Qt Source to c:\build\qt-everywhere-opensource-src-5.2.1
  • Add Python, Ruby, Perl, Jom to PATH
  • Add GnuWin32 to PATH
:: Add GnuWin32 to PATH
set PATH=%PATH%;c:\build\qt-everywhere-opensource-src-5.2.1\gnuwin32\bin

:: Setup Visual C++ Environment

"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" x86

:: Add ICU to INLCUDE, LIB, PATH

set INCLUDE=%INCLUDE%;c:\build\icu\dist\x86-static-release\include
set LIB=%LIB%;c:\build\icu\dist\x86-static-release\lib
set PATH=%PATH%;c:\build\icu\dist\x86-static-release\lib

:: Add OPENSSL to INLCUDE, LIB, PATH

set INCLUDE=%INCLUDE%;c:\build\\openssl\dist\x86-static-release\include
set LIB=%LIB%;c:\build\\openssl\dist\x86-static-release\lib
set PATH=%PATH%;c:\build\\openssl\dist\x86-static-release\lib

:: Create build directory

mkdir c:\build\qt-everywhere-opensource-src-5.2.1\build\x86-static-release
cd c:\build\qt-everywhere-opensource-src-5.2.1\build\x86-static-release

:: Configure

..\..\configure -prefix c:\build\qt-everywhere-opensource-src-5.2.1\dist\x86-static-release -platform win32-msvc2013 -static -release -c++11 -opensource -confirm-license -icu -qt-zlib -qt-pcre -qt-libpng -qt-libjpeg -qt-freetype -openssl-linked -skip qtwebkit -nomake tests -nomake examples -mp -make-tool jom OPENSSL_LIBS="-lssleay32 -llibeay32 -lgdi32 -lcrypt32 -luser32 -ladvapi32"

:: Build and install

jom -j8
jom install

:: HACK: copy required libs manually

cp C:\build\qt-everywhere-opensource-src-5.2.1\build\x86-static-debug\qtbase\lib\translator_*.lib c:\build\qt-everywhere-opensource-src-5.2.1\dist\x86-static-release\lib
cp C:\build\qt-everywhere-opensource-src-5.2.1\build\x86-static-debug\qtbase\lib\preprocessor*.lib c:\build\qt-everywhere-opensource-src-5.2.1\dist\x86-static-release\lib

::

The steps to build debug version and x64 versions are similar to the ones above.

Statically Link Against an LGPL'd Library in a Closed Source Project

Statically link against a LGPL’d library in a closed source project is possible, if object format of your application is provided. In that way, a user can modify the LGPL’d library and relink the application. See GPL FAQ and here for more details.

Thursday, February 13, 2014

Test If a Type Has a Member Function

Sometimes we want to handle types with some specific member functions specially. The followin examples shows how to do that using SFINAE and enable_if.

#include <iostream>
#include <type_traits>

using namespace std;

// SFINAE test
template<typename T>
class has_test {
    template<typename U, U> class check {};
    template<typename C> static char f(check<void(C::*)(int), &C::test>*);
    template<typename C> static long f(...);
public:
    static const bool value = (sizeof(f<T>(nullptr)) == sizeof(char));
};

template<typename T>
std::enable_if_t<has_test<T>::value, void> test(T& t) {
    cout << typeid(T).name() << " has a 'test' member function. " << endl;
}

template<typename T>
std::enable_if_t<!has_test<T>::value, void> test(T& t) {
    cout << typeid(T).name() << " has no 'test' member function. " << endl;
}

struct A {
    void test(int) {}
};

struct B {
};

int main() {
    A a;
    B b;
    test(a);
    test(b);
}

Here is the output of the program.

struct A has a 'test' member function. 
struct B has no 'test' member function.