40 lines
981 B
C
40 lines
981 B
C
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <curl/curl.h>
|
|
#include <sys/random.h>
|
|
|
|
#include "crawler.h"
|
|
#include "module.h"
|
|
#include "util.h"
|
|
|
|
const char *allowed_hosts[] = {
|
|
"32bit.cafe",
|
|
//"en.wikipedia.org",
|
|
NULL
|
|
};
|
|
|
|
int main(int argc, char **argv) {
|
|
if (argc != 2) {
|
|
fprintf(stderr, "url dumbass\n");
|
|
return 1;
|
|
}
|
|
|
|
int seed, rc;
|
|
if ((rc = getrandom(&seed, sizeof(seed), 0)) != sizeof(seed)) {
|
|
fatal("getrandom() failed with %d", rc);
|
|
return 1;
|
|
}
|
|
srandom(seed);
|
|
|
|
crawlerconfig_t config = { .allowedhosts = allowed_hosts, .req_interval_s = 0 };
|
|
curl_global_init(CURL_GLOBAL_DEFAULT);
|
|
dynarr_init(moduleentryp_dynarr_t, config.enabledmodules);
|
|
for (moduleentry_t *module = availmodules; module->name != NULL; module++)
|
|
dynarr_push(config.enabledmodules, module);
|
|
crawler(argv[1], &config);
|
|
dynarr_destroy(config.enabledmodules);
|
|
curl_global_cleanup();
|
|
|
|
return 0;
|
|
}
|