C语言爬取HTML-爬取壁纸文末附源码

前言：这学期计算机软件课程设计的其中一个题目是使用C语言爬取HTML，本打算使用C语言的CSpidr库来实现，但是因为它的依赖liburi没有找到在哪里安装，所以放弃了这个想法，使用的是curl以及libxml2这两个库，能够提供访问网页以及xpath解析的功能就够用了。

项目使用C语言爬取壁纸，爬取的网站是https://wallhaven.cc
开发环境使用的是Ubuntu22.04，编译器gcc 11.3,使用makefile管理项目

依赖库的安装：

sudo apt update
sudo apt install curl libxml2-dev

项目结构：

项目由两个文件组成，一个是main.c源代码，另一个是用于makefile编译的makefile文件。

├── main.c
└── Makefile

程序编译并执行完成之后项目目录如下：

img：存放爬取的图片
main main.o可执行文件
result.txt:图片页面的URL
url.txt:每张图片的url

流程图如下：

使用说明：

该程序使用make命令对程序进行编译

该程序可以通过命令行参数指定起始页和结束页，命令如下：

sudo ./main start_page end_page

爬取的壁纸
在这里插入图片描述
以后再也不缺壁纸啦

源码：

makefile

# CC = gcc
CFLAGS = -I/usr/include/libxml2
LDFLAGS = -lcurl -lxml2

TARGET = main
SRCS = main.c
OBJS = $(SRCS:.c=.o)

all: $(TARGET)

$(TARGET): $(OBJS)
	$(CC) $(CFLAGS) $(OBJS) -o $(TARGET) $(LDFLAGS)

%.o: %.c
	$(CC) $(CFLAGS) -c $< -o $@

clean:
	rm -f $(OBJS) $(TARGET)

源码

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <curl/curl.h>
#include <libxml/HTMLparser.h>
#include <libxml/xpath.h>
#include <dirent.h>
#include <sys/stat.h>
#include <unistd.h>

void get_imgpage_url(int start_page, int end_page); // 获取图片页面的url
void get_img_url();                                 // 获取图片的url
void download_img(char *url, char *outfilename);    // 下载图片

#define USER_AGENT "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36 Edg/104.0.1293.63"

typedef struct
{
    char *buffer;
    size_t size;
} MemoryStruct;

// 回调函数，将获取的数据写入缓冲区
size_t write_callback(void *contents, size_t size, size_t nmemb, void *userp)
{
    size_t real_size = size * nmemb;
    MemoryStruct *mem = (MemoryStruct *)userp;

    mem->buffer = realloc(mem->buffer, mem->size + real_size + 1);
    if (mem->buffer == NULL)
    {
        printf("Failed to allocate memory.\n");
        return 0;
    }

    memcpy(&(mem->buffer[mem->size]), contents, real_size);
    mem->size += real_size;
    mem->buffer[mem->size] = '\0';

    return real_size;
}

/// @brief 获取每一页的图片链接
/// @param start_page
/// @param end_page
void get_imgpage_url(int start_page, int end_page)
{
    CURL *curl;
    CURLcode res;
    MemoryStruct response;
    response.buffer = malloc(1);
    response.size = 0;

    printf("初始化libcurl...\n");
    curl_global_init(CURL_GLOBAL_DEFAULT); // 初始化libcurl
    printf("创建会话...\n");
    curl = curl_easy_init(); // 创建会话
    if (curl)
    {
        FILE *file = fopen("result.txt", "w"); // 打开文件以保存结果
        if (file == NULL)
        {
            printf("Failed to open file.\n");
            return;
        }

        // 循环获取每一页的图片链接
        for (int page = start_page; page <= end_page; ++page)
        {
            char url[100];                                                            // 保存URL
            snprintf(url, sizeof(url), "https://wallhaven.cc/toplist?page=%d", page); // 设置URL

            curl_easy_setopt(curl, CURLOPT_URL, url);                      // 设置URL
            curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); // 设置回调函数
            curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&response);  // 设置回调函数的参数

            printf("发起请求...\n");
            res = curl_easy_perform(curl); // 执行请求
            if (res != CURLE_OK)
            {
                printf("curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
                continue;
            }

            htmlDocPtr doc = htmlReadMemory(response.buffer, response.size, NULL, NULL, HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR); // 解析HTML
            if (doc == NULL)
            {
                printf("Failed to parse HTML.\n");
                continue;
            }

            xmlXPathContextPtr xpathCtx = xmlXPathNewContext(doc); // 创建XPath上下文
            if (xpathCtx == NULL)
            {
                printf("Failed to create XPath context.\n");
                xmlFreeDoc(doc);
                continue;
            }

            for (int i = 0; i < 20; i++) // 获取每页的20个图片链接
            {
                char xpath[] = "/html/body/main/div[1]/section/ul/li[%d]/figure/a/@href";
                snprintf(xpath, sizeof(xpath), "/html/body/main/div[1]/section/ul/li[%d]/figure/a/@href", i + 1);
                xmlXPathObjectPtr xpathObj = xmlXPathEvalExpression((xmlChar *)xpath, xpathCtx); // 评估XPath表达式
                if (xpathObj == NULL)
                {
                    printf("Failed to evaluate XPath expression.\n");
                    xmlXPathFreeContext(xpathCtx);
                    xmlFreeDoc(doc);
                    continue;
                }

                xmlNodeSetPtr nodes = xpathObj->nodesetval; // 获取结果
                for (int i = 0; i < nodes->nodeNr; ++i)
                {
                    xmlChar *href = xmlNodeListGetString(doc, nodes->nodeTab[i]->xmlChildrenNode, 1);
                    if (href != NULL)
                    {
                        fprintf(file, "%s\n", href); // 将结果写入文件
                        xmlFree(href);
                    }
                }
                xmlXPathFreeObject(xpathObj);
            }

            xmlXPathFreeContext(xpathCtx);
            xmlFreeDoc(doc);
            printf("Page %d 链接获取完成.\n", page);
        }

        fclose(file);
        free(response.buffer);
        curl_easy_cleanup(curl);
    }
    else
    {
        printf("curl_easy_init() failed.\n");
    }

    curl_global_cleanup();
    printf("开始解析图片链接...\n");
    get_img_url();
}
static size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream)
{
    size_t written = fwrite(ptr, size, nmemb, (FILE *)stream);
    return written;
}
void download_img(char *url, char *outfilename)
{
    printf("正在下载%s\n",outfilename);
    CURL *curl;
    FILE *fp;
    CURLcode res;
    // char url[256] = "https://w.wallhaven.cc/full/p9/wallhaven-p9w7l9.png";
    // char outfilename[256] = "image.png";

    curl_global_init(CURL_GLOBAL_DEFAULT);
    curl = curl_easy_init();

    if (curl)
    {
        curl_easy_setopt(curl, CURLOPT_USERAGENT, USER_AGENT);
        curl_easy_setopt(curl, CURLOPT_URL, url);
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);

        // 打开当前目录下的img文件夹，没有则创建
        DIR *dir = opendir("img");
        if (dir == NULL)
        {
            mkdir("img", 0777);
        }
        closedir(dir);
        // 打开img文件夹下的图片，没有则创建
        char img_path[256] = "img/";
        strcat(img_path, outfilename);
        fp = fopen(img_path, "wb");
        if (fp)
        {
            curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
            res = curl_easy_perform(curl);
            if (res != CURLE_OK)
            {
                fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
            }
            else
            {
                printf("Image saved successfully.\n");
            }
            fclose(fp);
        }
        else
        {
            fprintf(stderr, "Failed to create file: %s\n", outfilename);
        }

        curl_easy_cleanup(curl);
    }

    curl_global_cleanup();
}
void get_img_url()
{
    FILE *file;
    FILE *furl;
    char url[256];
    char outfilename[256];
    int count = 1;

    file = fopen("result.txt", "r");
    furl = fopen("url.txt", "w");
    if (file)
    {
        while (fgets(url, sizeof(url), file) != NULL)
        {
            // 去除换行符
            url[strcspn(url, "\n")] = '\0';

            //

            // 访问页面并获取内容
            CURL *curl = curl_easy_init();
            if (curl)
            {
                curl_easy_setopt(curl, CURLOPT_URL, url);
                curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);

                MemoryStruct html_content;
                html_content.buffer = malloc(1);
                html_content.size = 0;

                curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
                curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&html_content);
                // printf("start curl_easy_perform\n");
                CURLcode res = curl_easy_perform(curl);
                if (res != CURLE_OK)
                {
                    fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
                    curl_easy_cleanup(curl);
                    free(html_content.buffer);
                    return;
                }

                // 解析HTML内容
                // printf("start Parsing\n");
                xmlDocPtr doc = htmlReadMemory(html_content.buffer, html_content.size, "noname.html", NULL, HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING);
                xmlXPathContextPtr xpathCtx = xmlXPathNewContext(doc);

                // 执行XPath查询，获取图片URL
                char *xpathExpr = "//*[@id='wallpaper']/@src";
                xmlXPathObjectPtr xpathObj = xmlXPathEvalExpression((xmlChar *)xpathExpr, xpathCtx);
                xmlNodeSetPtr nodes = xpathObj->nodesetval;

                if (nodes != NULL && nodes->nodeNr > 0)
                {
                    xmlChar *imgUrl = xmlNodeListGetString(doc, nodes->nodeTab[0]->xmlChildrenNode, 1);
                    if (imgUrl != NULL)
                    {
                        // fwrite(imgUrl, strlen(imgUrl), 1, furl);// 将结果写入文件并加换行符
                        fprintf(furl, "%s\n", imgUrl);
                        printf("图片URL: %s\n", imgUrl);
                        // print_image_url((const char *)imgUrl);
                        xmlFree(imgUrl);
                    }
                    else
                    {
                        printf("imgUrl is NULL\n");
                    }
                }
                xmlXPathFreeObject(xpathObj);
                xmlXPathFreeContext(xpathCtx);
                xmlFreeDoc(doc);

                // free(html_content.buffer);
            }
            curl_easy_cleanup(curl);
        }
        printf("图片链接获取完成.\n准备下载\n");
    }
    fclose(file);
    fclose(furl);

    furl = fopen("url.txt", "r");
    // 读取url.txt文件，download图片
    if (furl)
    {
        count = 1;
        while (fgets(url, sizeof(url), furl) != NULL)
        {
            // 去除换行符
            url[strcspn(url, "\n")] = '\0';
            // 根据读取url的后3位判断图片的类型，并设置图片的后缀名，后缀名还需要有count序号
            char *suffix = url + strlen(url) - 3;
            if (strcmp(suffix, "jpg") == 0)
            {
                strcpy(outfilename, "image");
                char count_str[10];
                sprintf(count_str, "%d", count);
                strcat(outfilename, count_str);
                strcat(outfilename, ".jpg");
            }
            else if (strcmp(suffix, "png") == 0)
            {
                strcpy(outfilename, "image");
                char count_str[10];
                sprintf(count_str, "%d", count);
                strcat(outfilename, count_str);
                strcat(outfilename, ".png");
            }
            else if (strcmp(suffix, "gif") == 0)
            {
                strcpy(outfilename, "image");
                char count_str[10];
                sprintf(count_str, "%d", count);
                strcat(outfilename, count_str);
                strcat(outfilename, ".gif");
            }
            else
            {
                printf("图片类型错误\n");
                continue;
            }
            download_img(url, outfilename);
            count++;
        }
    }
    else
    {
        printf("Failed to open furl.\n");
    }
    fclose(furl);
}

int main(int argc, char const *argv[])
{
    if (argc != 3)
    {
        printf("Usage: %s <start_page> <end_page>\n", argv[0]);
        return 0;
    }
    else if (atoi(argv[1]) > atoi(argv[2]))
    {
        printf("<start_page> must 大于 <end_page>\n");
    }
    else
    {
        printf("start page: %s\n", argv[1]);
        printf("end page: %s\n", argv[2]);
        get_imgpage_url(atoi(argv[1]), atoi(argv[2]));
    }
    return 0;
}