VBoxGDB.cpp

Go to the documentation of this file.

// VBoxGDB.cpp : Defines the entry point for the console application.
//

#include "stdafx.h"
#include <stdio.h>
#include "../../GDBServerFoundation/GDBServer.h"
#include "../../GDBServerFoundation/GDBStub.h"
#include "../../GDBServerFoundation/GlobalSessionMonitor.h"
#include "VBoxClient.h"
#include "VBoxTarget.h"

using namespace BazisLib;
using namespace GDBServerFoundation;
using namespace VBoxGDB;

class SimpleStub : public BasicGDBStub
{
    virtual StubResponse HandleRequest(const BazisLib::TempStringA &requestType, char splitterChar, const BazisLib::TempStringA &requestData)
    {
        printf("%s%c%s\n", DynamicStringA(requestType).c_str(), splitterChar, DynamicStringA(requestData).c_str());
        return __super::HandleRequest(requestType, splitterChar, requestData);
    }

    virtual void OnProtocolError(const TCHAR *errorDescription)
    {
        _tprintf(_T("Protocol error: %s\n"), errorDescription);
    }
};

class StubWithLogging : public GDBStub
{
    FILE *pLogFile;

    virtual StubResponse HandleRequest(const BazisLib::TempStringA &requestType, char splitterChar, const BazisLib::TempStringA &requestData)
    {
        printf(">> %s%c%s\n", DynamicStringA(requestType).c_str(), splitterChar, DynamicStringA(requestData).c_str());
        fprintf(pLogFile, ">> %s%c%s\n", DynamicStringA(requestType).c_str(), splitterChar, DynamicStringA(requestData).c_str());
        StubResponse response = __super::HandleRequest(requestType, splitterChar, requestData);
        printf("<< %s\n", std::string(response.GetData(), response.GetSize()).c_str());
        fprintf(pLogFile, "<< %s\n", std::string(response.GetData(), response.GetSize()).c_str());
        fflush(pLogFile);
        return response;
    }

public:
    StubWithLogging(ISyncGDBTarget *pTarget)
        : GDBStub(pTarget, true)
    {
        pLogFile = fopen("gdbserver.log", "w");
    }

    ~StubWithLogging()
    {
        fclose(pLogFile);
    }
};

static volatile LONG s_bStubCreated;

class GDBStubFactory : public IGDBStubFactory
{
private:
    VBoxClient *m_pClient;

public:
    GDBStubFactory(VBoxClient *pClient)
        : m_pClient(pClient)
    {
    }

    virtual IGDBStub *CreateStub(GDBServer *pServer)
    {
        if (InterlockedExchange(&s_bStubCreated, 1))
            return NULL;

        printf("GDB connected.\n");
        pServer->StopListening();
        VBoxTarget32 *pTarget = new VBoxTarget32(m_pClient, false);
        //return new StubWithLogging(pTarget);
        return new GDBStub(pTarget);
    }

    virtual void OnProtocolError(const TCHAR *errorDescription)
    {
        _tprintf(_T("Protocol error: %s\n"), errorDescription);
    }
};

#pragma region The code that reproduces the VirtualBox hanging bug
static VBoxClient *pClt;

BOOL CALLBACK CtrlHandler( DWORD dwType )
{
    if (dwType == CTRL_C_EVENT)
    {
        pClt->RequestStopAsync();
        return TRUE;
    }
    return FALSE;
}

bool ReproduceVBoxBug(const char *pHost, int port)
{
    VBoxClient client(BazisLib::Network::DNS::GetHostByName(pHost).SetPort(port));
    if (!client.Initialize())
        return false;
    pClt = &client;

    client.RequestStopAsync();
    client.WaitForStop();
    ULONGLONG val;

    SetConsoleCtrlHandler(CtrlHandler, TRUE);

    //The loop will hang around iteration 704 or so. Definitely before 20K.
    //Commenting/uncommenting random command lines below changes the amount of iterations till hang
    client.ReadRegister("esp", &val);
    for (int i = 0;;i++)
    {
        client.RequestStopAsync();
        client.WaitForStop();

        //      client.ReadRegister("esp", &val);
        char sz[32], szCmd[128];
        size_t len = 4;
        sprintf(szCmd, "db %08I64x", val);
        //      if (!client.ReadMemory(val, sz, &len, false))
        //          __asm int 3;
        client.ExecuteCommand(szCmd);
        strcpy(szCmd, "showvars");
        //if (!(i % 5))
        client.ExecuteCommand(szCmd);

        printf("%d\n", i);
        //fflush(stdout);
    }

    return true;
}
#pragma endregion

void DisplayHelp()
{
    printf("Usage: VBoxGDB <VirtualBox debug port or host:port> [--gdbport=<GDBServer port>]\n");
}


static VBoxClient *s_pClient;

static BOOL CALLBACK safeCtrlHandler( DWORD dwType )
{
    if (dwType == CTRL_C_EVENT)
    {
        printf("Stopping target... If you intended to close this program, press Ctrl+Break.\n");
        if (s_pClient)
            s_pClient->RequestStopAsync();
        return TRUE;
    }
    if (dwType == CTRL_BREAK_EVENT|| dwType == CTRL_CLOSE_EVENT)
    {
        if (s_pClient)
            s_pClient->Disconnect();
        return FALSE;
    }
    return FALSE;
}


int main(int argc, char* argv[])
{
    printf("VBoxGDB v0.1 [http://sysprogs.com/VBoxGDB]\n");
    printf("This pre-release version has the following limitations:\n\
\t*Only 64-bit VirtualBox 4.1.x is supported\n\
\t*Only 32-bit Linux guests are supported\n\
\t*Only single-CPU virtual machines are supported\n\
---------------------------------------------------------------------\n");

    if (argc < 2)
    {
        DisplayHelp();
        return 1;
    }

    int vboxPort, gdbPort = 2000;
    std::string vboxHost = "localhost";

    char *p = strchr(argv[1], ':');
    if (!p)
        vboxPort = atoi(argv[1]);
    else
    {
        vboxHost = std::string(argv[1], p - argv[1]);
        vboxPort = atoi(p + 1);
    }

    for (int i = 2; i < argc; i++)
    {
        if (memicmp(argv[i], "--gdbport=", 10))
            gdbPort = atoi(argv[i] + 10);
        else if (!strcmp(argv[i], "--tryhang"))
            ReproduceVBoxBug(vboxHost.c_str(), vboxPort);
    }

    printf("Trying to connect to VirtualBox debugger at %s:%d...\n", vboxHost.c_str(), vboxPort);
    VBoxClient client(BazisLib::Network::DNS::GetHostByName(vboxHost.c_str()).SetPort(vboxPort));
    if (!client.Initialize())
    {
        printf("Cannot connect to VirtualBox debugger.\n");
        return 1;
    }

    s_pClient = &client;

    printf("Connected to VirtualBox debugger\n", vboxPort);

    GDBServer srv(new GDBStubFactory(&client));
    SetConsoleCtrlHandler(safeCtrlHandler, TRUE);
    printf("Listening for GDB connections on port %d\n", gdbPort);
    printf("To start debugging, run \"gdb vmlinux\" and execute \"target remote :%d\"\n", gdbPort, gdbPort);
    srv.Start(gdbPort);
    srv.WaitForTermination();

    s_pClient = NULL;
    return 0;
}