libcurl with Ada - how to HTTP POST transfer with large file upload

[Matt Borchers <mattborchers@gmail.com>]

A recent situation has caused some previously working code (written in Ada 95 era) to stop working.  What has changed is updating from using a very old compiler (GNAT 6.3.0) and AWS (Ada Web Server) v2.3.0 to the latest available.

We are using libcurl v7.19.6 (quite old) to transfer files using unauthenticated HTTP.  I have tried to build with the latest available libcurl library files.  Although the executable builds, the executable will not run.  So for now, at least, I am stuck with the v7.19.6.

We are also using the AdaCurl binding by Andreas Almroth that is dated 2005-05-24.  I have found a few errors with this binding recently in my investigation of how to fix my problem.

The files we transfer can be small to very large.  The old mechanism that was in place was this:  We set up a libcurl EASY session with PUT.  If the file size was less than 512MB then a buffer was created in memory and that buffer written to disk.  If the file was larger then we read the message body directly from the socket and wrote that data to disk.

The message body can no longer be read directly from the socket when using the new AWS.  I am told that in order to avoid a memory buffer that I should send the file as an attachment to a POST request.

My goal is to transfer any file with libcurl to AWS in the fastest way possible while avoiding a temporary memory buffer.

I have tried many things over the past week and have had no luck in getting libcurl to perform how I want it to so that AWS can retrieve the data from the message body.  The libcurl documentation and its debug routine option is lackluster in that it does not go in-depth enough to help people understand how to correct what is wrong.  I have tried coding up the many examples available but the examples don't work.  I undertand that when it is working properly the file will be written directly to disk and the HTTP response handler will be provided with the filename as a FORM parameter.

Does anybody have a working example of using libcurl with or without AWS as I described?  Perhaps a better alternative to libcurl for my situation?  Below is the latest code I have which I've read is on the right track.  I don't really understand the adacurl.multi package which is used below with the adacurl.easy package.

Thank you for any assistance.
Matt

declare
    hndl      : CURL_P;
    mhndl     : CURLM_P;
    mc        : CURLMcode;
    still_run : aliased INTERFACES.C.INT := 0;
    u32_str   : STRING := UNSIGNED_32'Image( u32 );
    ca        : aliased CHAR_ARRAY := to_c( "http://.../database/" & u32_str(2..u32_str'Last) );
    continue  : aliased CHAR_ARRAY := to_c( "Expect:" );
    rcode     : aliased LONG := 0;
    s500      : constant := 500;
    form,
    ftail     : CURL_HTTPPOST_P;
    fcode     : CURLFORMcode;
    dbid      : aliased CHAR_ARRAY := to_c( u32_str(2..u32_str'Last) );
    sendfile  : aliased CHAR_ARRAY := to_c( "sendfile" );
    filename  : aliased CHAR_ARRAY := to_c( "filename" );
    submit    : aliased CHAR_ARRAY := to_c( "submit" );
    send      : aliased CHAR_ARRAY := to_c( "send" );
begin
    hndl := curl_easy_init;
    if hndl = NULL then
        text_io.put_line( "Curl failed to initialize" );
        return;
    end if;
    mhndl := curl_multi_init;
    if mhndl = NULL then
        text_io.put_line( "Curl-Multi failed to initialize" );
        return;
    end if;

    code := curl_easy_setopt( hndl, CURLOPT_VERBOSE, LONG(1) );
    code := curl_easy_setopt( hndl, CURLOPT_DEBUGFUNCTION, curl_debug'Unrestricted_Access );
    code := curl_easy_setopt( hndl, CURLOPT_ERRORBUFFER, to_chars_ptr( errbuf'Unrestricted_Access ) );
    code := curl_easy_setopt( hndl, CURLOPT_READDATA, fd'Address );
    code := curl_easy_setopt( hndl, CURLOPT_READFUNCTION, wrap_fread'Unrestricted_Access );
    code := curl_easy_setopt( hndl, CURLOPT_WRITEFUNCTION, ignore_response'Unrestricted_Access );
    code := curl_easy_setopt( hndl, CURLOPT_POST, LONG(1) );
    code := curl_easy_setopt( hndl, CURLOPT_URL, to_chars_ptr( ca'Unrestricted_Access ) );
    if code /= CURLE_OK then
        text_io.put_line( "Failed to set url, code is " & CURLCODE'Image( code ) );
        return;
    end if;

    headers := curl_slist_append( headers, to_chars_ptr( continue'Unrestricted_Access ) );
    code    := curl_easy_setopt( hndl, CURLOPT_HTTPHEADER, headers.all'Address );

    fcode := curl_formadd( form'Unrestricted_Access, ftail'Unrestricted_Access,
       CURLFORM_COPYNAME, to_chars_ptr( sendfile'Unrestricted_Access ),
       CURLFORM_FILE, to_chars_ptr( dbid'Unrestricted_Access ),
       CURLFORM_END );

    fcode := curl_formadd( form'Unrestricted_Access, ftail'Unrestricted_Access,
       CURLFORM_COPYNAME, to_chars_ptr( filename'Unrestricted_Access ),
       CURLFORM_COPYCONTENTS, to_chars_ptr( dbid'Unrestricted_Access ),
       CURLFORM_END );

    fcode := curl_formadd( form'Unrestricted_Access, ftail'Unrestricted_Access,
       CURLFORM_COPYNAME, to_chars_ptr( submit'Unrestricted_Access ),
       CURLFORM_COPYCONTENTS, to_chars_ptr( send'Unrestricted_Access ),
       CURLFORM_END );

    code := curl_easy_setopt( hndl, CURLOPT_HTTPPOST, form.all'Address );

    mc := curl_multi_add_handle( mhndl, hndl );
    mc := curl_multi_perform( mhndl, still_run'Unrestricted_Access );
    while still_run /= 0 loop
        declare
            fdread,
            fdwrite,
            fdexcep : aliased FD_SET := (others => 0);
            maxfd   : aliased INTERFACES.C.INT := -1;

        begin
            mc := curl_multi_fdset( mhndl, fdread'Unrestricted_Access,
                                           fdwrite'Unrestricted_Access,
                                           fdexcep'Unrestricted_Access,
                                           maxfd'Unrestricted_Access );

            if mc /= CURLM_OK then
                text_io.put_line( "curl_multi_fdset() failed, code =" & mc'imaage );
                exit;
            end if;

            delay 0.1;

            mc := curl_multi_perform( mhndl, still_run'Unrestricted_Access );
            if mc = CURLM_OK then
                code := CURLE_OK;
            else
                code := CURL_LAST;
            end if;
        end;
    end loop;

    if code = CURLE_OK then
        code := curl_easy_getinfo( hndl, CURLINFO_RESPONSE_CODE, rcode'Access );
        if code = CURLE_OK and then rcode >= s500 then
            text_io.put_line( "Server error executing curl for PUT_FILE response code :" & rcode'Image );
            code := CURLE_SEND_ERROR;
        end if;
    end if;

    mc := curl_multi_cleanup( mhndl );
    curl_formfree( form );
    curl_slist_free_all( headers );
    curl_easy_cleanup( hndl );
end;