CR2RE Pipeline Reference Manual 1.6.8
hdrl_download.c
1/*
2 * hdrl_download.c
3 *
4 * Created on: Jan 28, 2022
5 * Author: agabasch
6 */
7
8/*
9 * This file is part of HDRL
10 * Copyright (C) 2022 European Southern Observatory
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
25 */
26
27/*-----------------------------------------------------------------------------
28 Includes
29 -----------------------------------------------------------------------------*/
30
31#ifdef HAVE_CONFIG_H
32#include <config.h>
33#endif
34
35#include <string.h>
36#include <cpl.h>
37#include <curl/curl.h>
38
39#include <stdio.h>
40#include <stdlib.h>
41#include <unistd.h>
42
43/*----------------------------------------------------------------------------*/
78/*----------------------------------------------------------------------------*/
79
82/***************************************************************************
83 * _ _ ____ _
84 * Project ___| | | | _ \| |
85 * / __| | | | |_) | |
86 * | (__| |_| | _ <| |___
87 * \___|\___/|_| \_\_____|
88 *
89 * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
90 *
91 * This software is licensed as described in the file COPYING, which
92 * you should have received as part of this distribution. The terms
93 * are also available at https://curl.se/docs/copyright.html.
94 *
95 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
96 * copies of the Software, and permit persons to whom the Software is
97 * furnished to do so, under the terms of the COPYING file.
98 *
99 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
100 * KIND, either express or implied.
101 *
102 ***************************************************************************/
103
104
105
106
107/* <DESC>
108 * Shows how the write callback function can be used to download data into a
109 * chunk of memory instead of storing it in a file.
110 * </DESC>
111 */
112struct MemoryStruct {
113 char *memory;
114 size_t size;
115};
116
117static size_t
118WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp)
119{
120 size_t realsize = size * nmemb;
121 struct MemoryStruct *mem = (struct MemoryStruct *)userp;
122
123 char *ptr = realloc(mem->memory, mem->size + realsize + 1);
124 if(!ptr) {
125 /* out of memory! */
126 cpl_error_set_message(cpl_func, CPL_ERROR_UNSPECIFIED,
127 "Not enough memory (realloc returned NULL)");
128 /*printf("not enough memory (realloc returned NULL)\n");*/
129 return 0;
130 }
131
132 mem->memory = ptr;
133 memcpy(&(mem->memory[mem->size]), contents, realsize);
134 mem->size += realsize;
135 mem->memory[mem->size] = 0;
136
137 return realsize;
138}
139
140
141
142
143/*----------------------------------------------------------------------------*/
159/*----------------------------------------------------------------------------*/
160char * hdrl_download_url_to_buffer (const char * url, size_t * data_length)
161{
162
163 /* Check and dump the input */
164 cpl_ensure (url, CPL_ERROR_NULL_INPUT, NULL);
165 cpl_ensure (data_length, CPL_ERROR_NULL_INPUT, NULL);
166
167 cpl_msg_debug (cpl_func, "Using URL %s", url);
168
169
170 /* Writing into a data buffer */
171
172 CURL *curl_handle;
173 CURLcode res;
174 struct MemoryStruct chunk;
175
176 chunk.memory = malloc(1); /* will be grown as needed by the realloc above */
177 chunk.size = 0; /* no data at this point */
178
179 curl_global_init(CURL_GLOBAL_ALL);
180
181 /* init the curl session */
182 curl_handle = curl_easy_init();
183
184 /* specify URL to get */
185 curl_easy_setopt(curl_handle, CURLOPT_URL, url);
186
187 /*
188 * A long parameter set to 1 tells the library to fail the request if the HTTP
189 * code returned is equal to or larger than 400. The default action would be
190 * to return the page normally, ignoring that code.
191 * This method is not fail-safe and there are occasions where non-successful
192 * response codes will slip through, especially when authentication is
193 * involved(response codes 401 and 407).
194 */
195 curl_easy_setopt(curl_handle, CURLOPT_FAILONERROR, 1L);
196
197 if(cpl_msg_get_level() == CPL_MSG_DEBUG){
198 /* Switch on full protocol/debug output while testing */
199 curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 1L);
200 /* disable progress meter, set to 0L to enable it */
201 curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1L);
202 }
203
204 /* send all data to this function */
205 curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
206
207 /* we pass our 'chunk' struct to the callback function */
208 curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk);
209
210 /* some servers do not like requests that are made without a user-agent
211 field, so we provide one */
212 curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0");
213
214 /* get it! */
215 res = curl_easy_perform(curl_handle);
216
217 /* check for errors */
218 if(res != CURLE_OK) {
219 /*
220 fprintf(stderr, "libcurl failed to retrieve: %s\n", curl_easy_strerror(res));
221 */
222 cpl_error_set_message(cpl_func, CPL_ERROR_DATA_NOT_FOUND,
223 "Could not retrieve data: %s",curl_easy_strerror(res));
224 curl_easy_cleanup(curl_handle);
225 curl_global_cleanup();
226 free(chunk.memory);
227 return NULL;
228 }
229 else {
230 /*
231 * Now, our chunk.memory points to a memory block that is chunk.size
232 * bytes big and contains the remote file.
233 *
234 * Do something nice with it!
235 */
236
237 *data_length = chunk.size;
238 curl_easy_cleanup(curl_handle);
239 curl_global_cleanup();
240 return chunk.memory;
241 }
242
243 }
244
245/* <DESC>
246 * Download a given URL into a local file.
247 * </DESC>
248 */
249
250static size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream)
251{
252 size_t written = fwrite(ptr, size, nmemb, (FILE *)stream);
253 return written;
254}
255
256/*----------------------------------------------------------------------------*/
273/*----------------------------------------------------------------------------*/
274
275cpl_error_code
276hdrl_download_url_to_file (const char * url, const char * filename)
277{
278 /* see also https://curl.se/libcurl/c/url2file.html */
279
280 /* Check and dump the input */
281 cpl_ensure_code(url, CPL_ERROR_NULL_INPUT);
282 cpl_ensure_code(filename, CPL_ERROR_NULL_INPUT);
283
284 cpl_msg_debug (cpl_func, "Using URL %s", url);
285 cpl_msg_debug (cpl_func, "Using File %s", filename);
286
287 CURL *curl_handle;
288 CURLcode res;
289 FILE *pagefile;
290
291 curl_global_init(CURL_GLOBAL_ALL);
292
293 /* init the curl session */
294 curl_handle = curl_easy_init();
295
296 /* set URL to get here */
297 curl_easy_setopt(curl_handle, CURLOPT_URL, url);
298
299 /*
300 * A long parameter set to 1 tells the library to fail the request if the HTTP
301 * code returned is equal to or larger than 400. The default action would be
302 * to return the page normally, ignoring that code.
303 * This method is not fail-safe and there are occasions where non-successful
304 * response codes will slip through, especially when authentication is
305 * involved(response codes 401 and 407).
306 */
307 curl_easy_setopt(curl_handle, CURLOPT_FAILONERROR, 1L);
308
309 if(cpl_msg_get_level() == CPL_MSG_DEBUG){
310 /* Switch on full protocol/debug output while testing */
311 curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 1L);
312 /* disable progress meter, set to 0L to enable it */
313 curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1L);
314 }
315
316 /* send all data to this function */
317 curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_data);
318
319 /* open the file */
320 pagefile = fopen(filename, "wb");
321
322 if(pagefile) {
323
324 /* write the page body to this file handle */
325 curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, pagefile);
326
327 /* get it! */
328 res = curl_easy_perform(curl_handle);
329
330 if(res != CURLE_OK) {
331 curl_easy_cleanup(curl_handle);
332 curl_global_cleanup();
333 /* close the header file */
334 fclose(pagefile);
335 return cpl_error_set_message(cpl_func, CPL_ERROR_DATA_NOT_FOUND,
336 "Could not retrieve data: %s",
337 curl_easy_strerror(res));
338 }
339
340 /* close the header file */
341 fclose(pagefile);
342 } else {
343 /* cleanup curl stuff */
344 curl_easy_cleanup(curl_handle);
345 curl_global_cleanup();
346 return cpl_error_set_message(cpl_func, CPL_ERROR_FILE_NOT_CREATED,
347 "The file %s could not be created",
348 filename);
349 }
350
351 /* cleanup curl stuff */
352 curl_easy_cleanup(curl_handle);
353 curl_global_cleanup();
354
355 return cpl_error_get_code();
356}
357
358
cpl_error_code hdrl_download_url_to_file(const char *url, const char *filename)
Downloads a url into a file on disc.
char * hdrl_download_url_to_buffer(const char *url, size_t *data_length)
Downloads a url into a c data buffer.