基于SSL(TLS)的HTTPS网页下载——如何编写健壮的可靠的网页下载

#define END_RESPONSE_HEADER        "rnrn"
#define CRLF                    "rn"

// 用于保存http响应的解析的相关参数
#define MAX_RESPONSE_HEADER_LEN        8196        // 响应头最大为8K
typedef struct http_params_st{
    BOOL bHeaderComplete;                                // 响应头数据是否接收完毕
    BOOL bMessageComplete;                                // 响应数据是否接收完毕
    BOOL bChunked;                                        // 传输方式是否为分块传输
    int  iStatusCode;                                    // HTTP响应码
    __int64 i64TotalReaded;                                // 一共读取的数据
    __int64 i64ContentLen;                                // Content-Length长度（响应头中解析出的"Content-Length"字段）
    __int64 i64BodyLen;                                    // 实际的body数据长度
                                                                    

    char szResponseHeader[MAX_RESPONSE_HEADER_LEN];        // 缓存HTTP响应头
    int iResponseHeaderLen;                                // 响应头的长度
    BOOL bResponseParsed;                                // 响应头是否已解析
    HANDLE hFile;                                        // 文件句柄，用于保存接收到的所有响应数据（原始数据）
    HANDLE hFileBody;                                    // 文件句柄，仅保存body数据
    map<string, string>    mapHeader;                        // 响应头中key=value对

    http_params_st(){
        iStatusCode = 0;
        bHeaderComplete = FALSE;
        bMessageComplete = FALSE;
        bChunked = FALSE;

        i64TotalReaded = 0;
        i64ContentLen = 0;
        i64BodyLen = 0;
        memset(szResponseHeader, 0, MAX_RESPONSE_HEADER_LEN);
        bResponseParsed = FALSE;
        iResponseHeaderLen = 0;
        hFile = NULL;
        hFileBody = NULL;
    }
}HTTP_PARAMS;

// 字符串去除头尾的空格
extern void StrTrim(char* pszSrc);
// 解析HTTP 响应头
extern BOOL ParseResponseHeader(HTTP_PARAMS* pHttpParams);
// 根据关键字获取对应的值
extern BOOL GetValueByKey(HTTP_PARAMS* pHttpParams, string strKey, string& strValue);



//=============================以下llhttp的回调函数=============================
// HTTP响应头读取完毕
static int on_llhttp_headers_complete(llhttp_t* llhttp)
{
    HTTP_PARAMS* pHttpParams = (HTTP_PARAMS*)llhttp->data;
    pHttpParams->bHeaderComplete = TRUE;
    
    return HPE_OK;
}

// HTTP响应读取完毕
static int on_llhttp_message_complete(llhttp_t* llhttp)
{
    HTTP_PARAMS* pHttpParams = (HTTP_PARAMS*)llhttp->data;
    pHttpParams->bMessageComplete = TRUE;

    return HPE_OK;
}

// llhttp上抛的body数据
static int on_llhttp_body(llhttp_t* llhttp, const char *at, size_t length)
{
    HTTP_PARAMS* pHttpParams = (HTTP_PARAMS*)llhttp->data;
    
    pHttpParams->i64BodyLen += length;
    if(INVALID_HANDLE_VALUE != pHttpParams->hFileBody && NULL != pHttpParams->hFileBody)
    {
        DWORD dwWrited = 0;
        ::WriteFile(pHttpParams->hFileBody, at, length, &dwWrited, NULL);
    }
    return HPE_OK;
}


//=============================以下为SSL层返回的业务数据=============================
static int OnSSLHttpDataNotify(const BYTE* pData, int iDataLen, DWORD dwCallbackData1, DWORD dwCallbackData2)
{
    if(NULL == pData || iDataLen <= 0)
        return SSL_DATA_RECV_FAILED;

    llhttp_t* llhttp = (llhttp_t*)dwCallbackData1;                // 来自SSL通信的用户自定义数据，此案例中为llhttp解析器
    HTTP_PARAMS* pHttpParams = (HTTP_PARAMS*)llhttp->data;        // 来自llhttp的用户自定义数据
    pHttpParams->i64TotalReaded += iDataLen;                    // 计算一共读取的数据

    // 将接收到的数据写入文件，这是原始数据，包含响应头
    // 数据内容可能是chunked，因此需要进一步解析
    DWORD dwWrited = 0;
    ::WriteFile(pHttpParams->hFile, pData, iDataLen, &dwWrited, NULL);


    // 调用llhttp进行解析
    int iRet = llhttp_execute(llhttp, (const char*)pData, iDataLen);
    if(HPE_OK != iRet)
        return SSL_DATA_RECV_FAILED;        // 通知SSL层：业务层发生错误，SSLRecv函数将返回        

    // 将数据缓存到pHttpParams->szResponseHeader
    if(0 == pHttpParams->iResponseHeaderLen)
    {
        
        if(pHttpParams->i64TotalReaded > MAX_RESPONSE_HEADER_LEN)
        {
            int iTotalReaded = int(pHttpParams->i64TotalReaded);
            int iPreReaded = iTotalReaded - iDataLen; // 之前读取的长度
            if(iPreReaded < MAX_RESPONSE_HEADER_LEN)
                memcpy(pHttpParams->szResponseHeader+iPreReaded, pData, MAX_RESPONSE_HEADER_LEN-iPreReaded);
            pHttpParams->iResponseHeaderLen = MAX_RESPONSE_HEADER_LEN;
        }
        else
        {
            int iTotalReaded = int(pHttpParams->i64TotalReaded);
            memcpy(pHttpParams->szResponseHeader+iTotalReaded-iDataLen, pData, iDataLen);
            pHttpParams->iResponseHeaderLen = iTotalReaded;
        }
    }
    
    // 计算HTTP响应头的长度
    if(!pHttpParams->bHeaderComplete)
    {
        // 缓冲区已满但没发现头，说明响应头太大超过8K，防止恶意攻击
        if(MAX_RESPONSE_HEADER_LEN == pHttpParams->iResponseHeaderLen)
        {
            printf("Too large HTTP response header.rn");
            return SSL_DATA_RECV_FAILED;
        }
    }
    else
    {
        // 如果没有解析HTTP响应头，则进行解析
        if(!pHttpParams->bResponseParsed)
        {
            // 查找"rnrn"
            char* pszResponseHeader = pHttpParams->szResponseHeader;
            char* pszFind = strstr(pszResponseHeader, END_RESPONSE_HEADER);    
            int iPos = pszFind - pszResponseHeader;
            pHttpParams->iResponseHeaderLen = iPos + 4;            // 计算真实的响应头长度，包含4字节的"rnrn"
            *(pszResponseHeader+pHttpParams->iResponseHeaderLen) = 0;
            pHttpParams->bResponseParsed = TRUE;
            pHttpParams->iStatusCode = llhttp->status_code;

            // 解析HTTP响应头
            ParseResponseHeader(pHttpParams);

            // 获取Content-Length长度
            string strValue;
            if(GetValueByKey(pHttpParams, "Content-Length", strValue))
            {
                pHttpParams->i64ContentLen = ::_atoi64(strValue.c_str());
            }
            else
            {
                pHttpParams->i64ContentLen = -1;    // 没有Content-Length字段
            }

            // 获取Transfer-Encoding编码方式，是否为chunked分块传输
            pHttpParams->bChunked = FALSE;
            if(GetValueByKey(pHttpParams, "Transfer-Encoding", strValue))
            {
                if(0 == _stricmp(strValue.c_str(), "chunked"))
                    pHttpParams->bChunked = TRUE;
            }
            // HTTP response头中既没有Content-Length字段，也没有Chunked字段，因此无法明确后续内容大小
            if(pHttpParams->i64ContentLen < 0 && !pHttpParams->bChunked)
                return SSL_DATA_RECV_FAILED;

        }
    }

    // 业务层数据全部读取完毕
    if(pHttpParams->bMessageComplete)
    {
        // 关闭文件
        return SSL_DATA_RECV_FINISHED;        // 通知SSL层：数据接收完毕，SSLRecv函数将返回TRUE
    }
    return SSL_DATA_RECV_STILL;                // 通知SSL层：继续接收数据，SSLRecv函数将继续接收服务器数据
}

// HTTPS协议测试
int _tmain(int argc, _TCHAR* argv[])
{
    // 加载sslite.dll
    CSSLWrap sslWrap;
    if(!sslWrap.Load())
    {
        printf("Load sslite.dll failed!rn");
        return -1;
    }
    printf("Load sslite.dll successfully!rn");

    // 获取ISSLSession接口
    ISSLSession* pSSLSession = sslWrap.GetSSLSession();
    
    //const char* pszServer = "www.sina.com.cn";
    //const char* pszServer = "www.baidu.com";    
    const char* pszServer =  "www.163.com";        // chunked
    int iRet = 0;
    // 建立SSL会话，也可以调用SSLConnect后再调用SSLHandShake来实现SSL会话
    if(!pSSLSession->SSLEstablish(pszServer, 443, iRet))
    {
        if(SSL_RET_CONNECT == iRet)
        {
            printf("Connect %s failed!rn", pszServer);
        }
        else if(SSL_RET_HANDSHAKE == iRet)
        {
            printf("SSL handshake failed!rn");
        }
        return -1;
    }
    // 建立连接后，显示当前的加密套件名称和ECC（椭圆加密）的组名称
    printf("SSL Session Established.rn");
    printf("Cipher Name: %srn", pSSLSession->SSLGetCipherName());
    printf("ECC Group Name: %srn", pSSLSession->SSLGetECGroupName());
    printf("Start HTTP communication.......rnrn");

    
    // 发送HTTP请求
    string strRequest;
    strRequest = "GET / HTTP/1.1rn";
    strRequest += "Accept: */*rn";
    strRequest += "Connection: Closern";
    //strRequest += "Accept-Encoding: gzip; brrn";    // 不支持压缩
    strRequest += "Host: ";
    strRequest += pszServer;
    strRequest += "rnrn";
    if(!pSSLSession->SSLSend((BYTE*)strRequest.c_str(), strRequest.length()))
    {
        printf("ERROR: SSLSend.rn");
        return -1;
    }

    /*
     接收HTTP响应数据
     1、iBuffSize将返回实际接收到的数据大小；
     2、如果接收的数据大于输入缓存arrBuff的尺寸，SSLRecv只会填满arrBuff缓存，
        后续数据将被丢弃。
     3、OnSSLHttpDataNotify，回调函数，业务层需要在回调函数中处理具体的业务数据，
        在本例中，使用开源的llhttp处理HTTP响应数据，如解析HTTP响应头，获取
        Content-Length字段大小或chunk，从而判断出后续要接收实际数据的尺寸。
        从而在llhttp的回调函数中通知上层用户。

        OnSSLHttpDataNotify返回值如下：
        3.1、SSL_DATA_RECV_STILL：业务层数据尚未读完，SSLRecv内部需要继续读取；
        3.2、SSL_DATA_RECV_FAILED：业务层出现错误，SSLRecv函数将返回FALSE；
        3.3、SSL_DATA_RECV_FINISHED：业务层数据处理完毕，SSLRecv函数将返回TRUE;
           本例中需要判断Content-Length来决定，业务层数据是否读取完毕。
      注：node.js中使用llhttp进行http数据解析，从而大幅提升解析效率
    */
    // 构造llhttp解析器，用于解析HTTP返回的响应数据
    llhttp_t llhttp_parser;
    llhttp_settings_t settings;
    llhttp_settings_init(&settings);
    settings.on_headers_complete = on_llhttp_headers_complete;        // http响应头已接收完毕通知
    settings.on_message_complete = on_llhttp_message_complete;        // http响应消息接收完毕
    settings.on_body = on_llhttp_body;                                // http除响应头外的消息体数据
    llhttp_init(&llhttp_parser, HTTP_RESPONSE, &settings);
    HTTP_PARAMS http_params;
    llhttp_parser.data = (void*)&http_params;    // 用户自定义数据


    BYTE arrBuff[1024] = {0};
    int iBuffSize = 1024;
    // 将读取到的所有响应内容保存到文件中，SSL层上抛的数据
    const char* pszPathFile = "C:/TestSSLHttp.html";
    http_params.hFile = ::CreateFile(pszPathFile, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS,
                                        FILE_ATTRIBUTE_NORMAL, NULL);
    if(INVALID_HANDLE_VALUE == http_params.hFile)
    {
        printf("ERROR: CreateFile "%s".rn", pszPathFile);
        return -1;
    }
    // 将读取到的Body内容保存到文件中，llhttp处理后的真实body数据
    const char* pszPathFileBody = "C:/TestSSLHttp_body.html";
    http_params.hFileBody = ::CreateFile(pszPathFileBody, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS,
                                        FILE_ATTRIBUTE_NORMAL, NULL);
    if(INVALID_HANDLE_VALUE == http_params.hFileBody)
    {
        printf("ERROR: CreateFile "%s".rn", pszPathFileBody);
        return -1;
    }
    
    BOOL bRet = pSSLSession->SSLRecv(arrBuff, iBuffSize, OnSSLHttpDataNotify, (DWORD)&llhttp_parser, 0);
    if(!bRet)
    {
        printf("ERROR: SSLRecv.rn");
    }

    ::CloseHandle(http_params.hFile);
    ::CloseHandle(http_params.hFileBody);
    
    printf("rn====================HTTP Response Header====================rn");
    printf("%s", http_params.szResponseHeader);

    printf("rn====================HTTP Response Save To File====================rn");
    printf("Write all response data to file: "%s"rn", pszPathFile);
    printf("Write body data to file: "%s"rn", pszPathFileBody);

    printf("rn====================HTTP Response Finished====================rn");
    if(!http_params.bChunked)
    {
        printf("Total Readed = %I64urnResponse Header Length = %drnContent Length = %I64urnContent-Length = %I64urnBody Length=%I64urn", 
            http_params.i64TotalReaded, http_params.iResponseHeaderLen,
            http_params.i64TotalReaded-http_params.iResponseHeaderLen, 
            http_params.i64ContentLen, http_params.i64BodyLen);
    }
    else
    {
        printf("Total Readed = %I64urnResponse Header Length = %drnContent Length = %I64urnTransfer-Encoding = chunkedrnBody Length = %I64urn", 
            http_params.i64TotalReaded, http_params.iResponseHeaderLen,
            http_params.i64TotalReaded-http_params.iResponseHeaderLen, 
            http_params.i64BodyLen);
    }
    // ！！释放ISSLSession接口
    sslWrap.ReleaseSSLSession(pSSLSession);

    printf("rnPress any key exit.....rn");
    getchar();
    return 0;
}

//=============================以下为公共函数=============================
// 字符串去除头尾的空格
void StrTrim(char* pszSrc) 
{
    if(NULL == pszSrc)
        return;

    int i = 0, j = 0;
    // 找到第一个非' '字符
    while (pszSrc[j] == ' ') {
        ++j;
    }

    // 如果字符串全为空
    if (pszSrc[j] == 0) {
        pszSrc[0] = 0;
        return;
    }

    int iIdx = j;        // 记录第一个非空字符位置
    int iStop = 0;
    while (pszSrc[j] != 0)
    {
        if (pszSrc[j] == ' ' && iStop == 0) {
            iStop = j;            // 记录后面遇到的一个空字符
        } 
        else if (pszSrc[j] != ' ' && iStop != 0) {
            iStop = 0;
        }
        // 将当前非空字符拷贝到以0为开始的新位置
        pszSrc[i++] = pszSrc[j++];
    }

    if (iStop > 0) {
        pszSrc[iStop - iIdx] = 0;
    } 
    else if (j != i) {
        pszSrc[i] = 0;
    }
}

// 解析HTTP 响应头
BOOL ParseResponseHeader(HTTP_PARAMS* pHttpParams)
{
    if(NULL == pHttpParams)
        return FALSE;
    int iLen = strlen(pHttpParams->szResponseHeader);
    char* pszResponseHeader = new char[iLen+1];
    strcpy(pszResponseHeader, pHttpParams->szResponseHeader);

    // 逐行解析
    int iPos = 0;
    char* pszKeyValue = pszResponseHeader;
    char* pszFind = strstr(pszKeyValue, CRLF);
    while(pszFind)
    {
        iPos = pszFind-pszKeyValue;
        *(pszKeyValue+iPos) = 0;
        if(0 == strlen(pszKeyValue))
            break;

        // 查找":"，并解析key:Value，存放于mapHeader中，便于后续使用
        char* pszColon = strstr(pszKeyValue, ":");
        if(pszColon)
        {
            int iPosColon = pszColon - pszKeyValue;
            *(pszKeyValue+iPosColon) = 0;
            char* pszKey = pszKeyValue;
            char* pszValue = pszKeyValue + iPosColon + 1; // SKip Colon

            // 去除头尾空格
            StrTrim(pszKey);
            StrTrim(pszValue);

            // 保存到map中
            string strKey = pszKey;
            string strValue = pszValue;
            map<string, string>::iterator iter = pHttpParams->mapHeader.find(strKey);
            if(iter == pHttpParams->mapHeader.end())
            {
                pHttpParams->mapHeader.insert(map<string, string>::value_type(strKey, strValue));
            }
            else
            {
                iter->second += ";";
                iter->second += strValue;
            }

        }

        // 查找下一行
        pszKeyValue = pszKeyValue + iPos + 2;    // Skip "rn"
        pszFind = strstr(pszKeyValue, CRLF);
    }

    delete pszResponseHeader;
    return TRUE;
}

// 根据关键字获取对应的值
BOOL GetValueByKey(HTTP_PARAMS* pHttpParams, string strKey, string& strValue)
{
    // 下面方法回出现由于key关键字的大小写不一，导致无法检索到
    //map<string, string>::iterator iter = pHttpParams->mapHeader.find(strKey);
    //if(iter == pHttpParams->mapHeader.end())
    //{
    //return FALSE;
    //}
    //strValue = iter->second;
    //return TRUE;

    map<string, string>::iterator    iter;
    for(iter = pHttpParams->mapHeader.begin(); iter != pHttpParams->mapHeader.end(); ++iter)
    {
        if(0 == _stricmp(iter->first.c_str(), strKey.c_str()))
        {
            strValue = iter->second;
            return TRUE;
        }
    }
    return FALSE;
}