Spider2/include/http.h

47 lines
1.2 KiB
C
Raw Normal View History

2024-09-01 16:12:11 +02:00
#ifndef __HTTP_H_
#define __HTTP_H_
#include <curl/curl.h>
#include <stdbool.h>
#include "util.h"
#include "module.h"
typedef struct {
int status, flags, num_requests;
} headercb_data_t;
typedef struct {
byte *base, *begin, *end;
moduleentryp_dynarr_t *modules;
const char *url;
bool wasrequested;
} writecb_data_t;
typedef struct {
writecb_data_t writecb_data;
headercb_data_t headercb_data;
} cbdata_t;
#define HEADERCB_VALID_MIME (1 << 0)
#define HEADERCB_CONTENT_TYPE_ENCOUNTERED (1 << 1)
headercb_data_t http_get_to_buf(CURLcode *res, CURL *curl, byte **cnt, size_t *cntlen);
char *relative2absolute(CURLU *curl_url_h, const char *parent, const char *relative);
charp_dynarr_t parsehrefs(CURLU *curl_url_h, const char *url, const char *page, size_t pagelen);
extern const char *useragents[];
typedef struct {
const char *host;
struct curl_slist *headers;
int totalfailurecnt, failurecnt, visitcnt;
} hostentry_t;
dynarr_def(hostentry_t, host_dynarr_t);
CURL *makehandle(const char *url, hostentry_t *host_entry, cbdata_t *cbdata, bool wasrequested);
void initcbdata(const char *url, moduleentryp_dynarr_t *modules, cbdata_t *data);
#endif