47 lines
1.2 KiB
C
47 lines
1.2 KiB
C
|
#ifndef __HTTP_H_
|
||
|
|
#define __HTTP_H_
|
||
|
|
|
||
|
|
#include <curl/curl.h>
|
||
|
|
#include <stdbool.h>
|
||
|
|
|
||
|
|
#include "util.h"
|
||
|
|
#include "module.h"
|
||
|
|
|
||
|
|
typedef struct {
|
||
|
|
int status, flags, num_requests;
|
||
|
|
} headercb_data_t;
|
||
|
|
|
||
|
|
typedef struct {
|
||
|
|
byte *base, *begin, *end;
|
||
|
|
moduleentryp_dynarr_t *modules;
|
||
|
|
const char *url;
|
||
|
|
bool wasrequested;
|
||
|
|
} writecb_data_t;
|
||
|
|
|
||
|
|
typedef struct {
|
||
|
|
writecb_data_t writecb_data;
|
||
|
|
headercb_data_t headercb_data;
|
||
|
|
} cbdata_t;
|
||
|
|
|
||
|
|
#define HEADERCB_VALID_MIME (1 << 0)
|
||
|
|
#define HEADERCB_CONTENT_TYPE_ENCOUNTERED (1 << 1)
|
||
|
|
|
||
|
|
headercb_data_t http_get_to_buf(CURLcode *res, CURL *curl, byte **cnt, size_t *cntlen);
|
||
|
|
char *relative2absolute(CURLU *curl_url_h, const char *parent, const char *relative);
|
||
|
|
charp_dynarr_t parsehrefs(CURLU *curl_url_h, const char *url, const char *page, size_t pagelen);
|
||
|
|
|
||
|
|
extern const char *useragents[];
|
||
|
|
|
||
|
|
typedef struct {
|
||
|
|
const char *host;
|
||
|
|
struct curl_slist *headers;
|
||
|
|
int totalfailurecnt, failurecnt, visitcnt;
|
||
|
|
} hostentry_t;
|
||
|
|
|
||
|
|
dynarr_def(hostentry_t, host_dynarr_t);
|
||
|
|
|
||
|
|
CURL *makehandle(const char *url, hostentry_t *host_entry, cbdata_t *cbdata, bool wasrequested);
|
||
|
|
void initcbdata(const char *url, moduleentryp_dynarr_t *modules, cbdata_t *data);
|
||
|
|
|
||
|
|
#endif
|