从 Rust 中的 bytes::Bytes 获取 C FILE 指针

Question

我想使用 Rust 中的 ecCodes 库读取从服务器下载的 GRIB 文件。但是，我当前的解决方案会导致分段错误。复制问题的提取示例如下。

我使用reqwest crate 下载文件并使用bytes() 得到Bytes¹ 的响应。要使用 ecCodes 读取文件，我需要使用 codes_grib_handle_new_from_file()² 创建一个 codes_handle，它作为参数需要 *FILE 通常来自 fopen()。但是，我想跳过 IO 操作。所以我想我可以使用libc::fmemopen() 从Bytes 获取*FILE。但是当我将*mut FILE 从fmemopen() 传递到codes_grib_handle_new_from_file() 时，就会出现分段错误。

我怀疑问题是当我从Bytes 获得*mut c_void 所需的fmemopen() 时。我想我可以这样做：

//get a *mut c_void pointer fom Bytes
//file has &Bytes type
let mut buf = BytesMut::from(file.as_ref());
let ptr = buf.as_mut_ptr();
let ptr = ptr as *mut c_void;

因为*mut 是必需的，所以我创建了BytesMut，然后我可以从中获取mut 指针。我认为这些转换是有问题的，因为在调试器信息中ptr 包含的内存地址与file 的ptr 字段不同。

对同一文件使用从libc::fopen() 获取的*FILE 不会导致段错误。所以问题出在fmemopen()附近。

ecCodes 库已正确构建（通过所有测试并在 C 中工作）和链接（调用堆栈中的调用正确）。

完整提取示例：

#![allow(unused)]
#![allow(non_camel_case_types)]

use bytes::{Bytes, BytesMut};
use libc::{c_char, c_void, fmemopen, size_t, FILE};
use reqwest;
use tokio;

// generated by bindgen
#[repr(C)]
#[derive(Debug, Copy, Clone)]
pub struct codes_handle {
    _unused: [u8; 0],
}

// generated by bindgen
#[repr(C)]
#[derive(Debug, Copy, Clone)]
pub struct codes_context {
    _unused: [u8; 0],
}

#[tokio::main]
async fn main() {
    // download the grib file from server
    // then get response as bytes
    let url = "https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.20210612/00/atmos/gfs.t00z.pgrb2.1p00.f000";
    let file = reqwest::get(url).await.unwrap().bytes().await.unwrap();

    // get Bytes from *FILE with fmemopen
    // file must outlive the pointer so it is borrowed here
    let file_handle = open_with_fmemopen(&file);

    let grib_handle = open_with_codes(file_handle);
}

pub fn open_with_fmemopen(file: &Bytes) -> *mut FILE {
    // size of buffer and mode to be read with
    let size = file.len() as size_t;
    let mode = "r".as_ptr() as *const c_char;

    // get a *mut c_void pointer fom Bytes
    let mut buf = BytesMut::from(file.as_ref());
    let ptr = buf.as_mut_ptr();
    let ptr = ptr as *mut c_void;

    // get *FILE with fmemopen
    let obj;
    unsafe {
        obj = fmemopen(ptr, size, mode);
    }
    
    obj
}

pub fn open_with_codes(file_handle: *mut FILE) -> *mut codes_handle {
    
    // default context for ecCodes
    let context: *mut codes_context = std::ptr::null_mut();

    // variable to hold error code
    let mut error: i32 = 0;

    // get codes_handle from *FILE
    let grib_handle;
    unsafe {
        // segmentation fault occurs here
        grib_handle = codes_grib_handle_new_from_file(context, file_handle, &mut error as *mut i32);
    }

    grib_handle
}

// binding to ecCodes C library
#[link(name = "eccodes")]
extern "C" {
    pub fn codes_grib_handle_new_from_file(
        c: *mut codes_context,
        f: *mut FILE,
        error: *mut i32,
    ) -> *mut codes_handle;
}

而且因为该示例可能需要相当大的努力来设置，所以我还附加了来自 GDB 的 seg 错误的调用堆栈：

__memmove_avx_unaligned_erms 0x00007f738b415fa6
fmemopen_read 0x00007f738b31c9b4
_IO_new_file_underflow 0x00007f738b31fd51
__GI___underflow 0x00007f738b32142e
__GI___underflow 0x00007f738b32142e
__GI__IO_default_xsgetn 0x00007f738b32142e
__GI__IO_fread 0x00007f738b312493
stdio_read 0x00007f738bb8db37
_read_any 0x00007f738bb8cf1b
read_any 0x00007f738bb8cfa3
_wmo_read_any_from_file_malloc 0x00007f738bb8e6f7
wmo_read_grib_from_file_malloc 0x00007f738bb8e7d7
grib_handle_new_from_file_no_multi 0x00007f738bb872a2
grib_new_from_file 0x00007f738bb8678f
grib_handle_new_from_file 0x00007f738bb85998
codes_grib_handle_new_from_file 0x00007f738bb8532b
example::open_with_codes main.rs:68
example::main::{{closure}} main.rs:34
core::future::from_generator::{{impl}}::poll<generator-0> mod.rs:80
tokio::park::thread::{{impl}}::block_on::{{closure}}<core::future::from_generator::GenFuture<generator-0>> thread.rs:263
tokio::coop::with_budget::{{closure}}<core::task::poll::Poll<()>,closure-0> coop.rs:106
std::thread::local::LocalKey<core::cell::Cell<tokio::coop::Budget>>::try_with<core::cell::Cell<tokio::coop::Budget>,closure-0,core::task::poll::Poll<()>> local.rs:272
std::thread::local::LocalKey<core::cell::Cell<tokio::coop::Budget>>::with<core::cell::Cell<tokio::coop::Budget>,closure-0,core::task::poll::Poll<()>> local.rs:248
tokio::coop::with_budget<core::task::poll::Poll<()>,closure-0> coop.rs:99
tokio::coop::budget<core::task::poll::Poll<()>,closure-0> coop.rs:76
tokio::park::thread::CachedParkThread::block_on<core::future::from_generator::GenFuture<generator-0>> thread.rs:263
tokio::runtime::enter::Enter::block_on<core::future::from_generator::GenFuture<generator-0>> enter.rs:151
tokio::runtime::thread_pool::ThreadPool::block_on<core::future::from_generator::GenFuture<generator-0>> mod.rs:71
tokio::runtime::Runtime::block_on<core::future::from_generator::GenFuture<generator-0>> mod.rs:452
example::main main.rs:34
core::ops::function::FnOnce::call_once<fn(),()> function.rs:227
std::sys_common::backtrace::__rust_begin_short_backtrace<fn(),()> backtrace.rs:125
std::rt::lang_start::{{closure}}<()> rt.rs:66
core::ops::function::impls::{{impl}}::call_once<(),Fn<()>> function.rs:259
std::panicking::try::do_call<&Fn<()>,i32> panicking.rs:379
std::panicking::try<i32,&Fn<()>> panicking.rs:343
std::panic::catch_unwind<&Fn<()>,i32> panic.rs:431
std::rt::lang_start_internal rt.rs:51
std::rt::lang_start<()> rt.rs:65
main 0x0000560f1d93c76c
__libc_start_main 0x00007f738b2bb565
_start 0x0000560f1d935f0e

---

¹来自bytes crate，而不是std::io
²函数返回的grib_handle只是codes_handle的别名

Answer 1

1- 尝试改变

let mode = "r".as_ptr() as *const c_char;

到

let mode = "r\0".as_ptr() as *const c_char;

Rust 的 &str 不是以 null 结尾的，而您将它传递给 C 中的字符串文字应该以 null 结尾。

2- 为 open_with_fmemopen 尝试以下实现：

pub fn open_with_fmemopen(file: &Bytes) -> *mut FILE {
    unsafe {
        let obj = fmemopen(file.as_ref() as *const _ as _, file.len(), "r\0".as_ptr() as _);
        obj
    }
}