Mise à jour de Monitor.py et autres scripts
This commit is contained in:
Debian
@@ -0,0 +1,364 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# distutils: language = c++
|
||||
|
||||
from pyarrow.includes.common cimport *
|
||||
from pyarrow.includes.libarrow cimport *
|
||||
from pyarrow.includes.libarrow_python cimport CTimePoint
|
||||
|
||||
cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil:
|
||||
|
||||
ctypedef enum CFileType "arrow::fs::FileType":
|
||||
CFileType_NotFound "arrow::fs::FileType::NotFound"
|
||||
CFileType_Unknown "arrow::fs::FileType::Unknown"
|
||||
CFileType_File "arrow::fs::FileType::File"
|
||||
CFileType_Directory "arrow::fs::FileType::Directory"
|
||||
|
||||
cdef cppclass CFileInfo "arrow::fs::FileInfo":
|
||||
CFileInfo()
|
||||
CFileInfo(CFileInfo)
|
||||
CFileInfo& operator=(CFileInfo)
|
||||
CFileInfo(const CFileInfo&)
|
||||
CFileInfo& operator=(const CFileInfo&)
|
||||
|
||||
CFileType type()
|
||||
void set_type(CFileType type)
|
||||
c_string path()
|
||||
void set_path(const c_string& path)
|
||||
c_string base_name()
|
||||
int64_t size()
|
||||
void set_size(int64_t size)
|
||||
c_string extension()
|
||||
CTimePoint mtime()
|
||||
void set_mtime(CTimePoint mtime)
|
||||
|
||||
cdef cppclass CFileSelector "arrow::fs::FileSelector":
|
||||
CFileSelector()
|
||||
c_string base_dir
|
||||
c_bool allow_not_found
|
||||
c_bool recursive
|
||||
|
||||
cdef cppclass CFileLocator "arrow::fs::FileLocator":
|
||||
shared_ptr[CFileSystem] filesystem
|
||||
c_string path
|
||||
|
||||
cdef cppclass CFileSystem "arrow::fs::FileSystem":
|
||||
shared_ptr[CFileSystem] shared_from_this()
|
||||
c_string type_name() const
|
||||
CResult[c_string] NormalizePath(c_string path)
|
||||
CResult[c_string] MakeUri(c_string path)
|
||||
CResult[CFileInfo] GetFileInfo(const c_string& path)
|
||||
CResult[vector[CFileInfo]] GetFileInfo(
|
||||
const vector[c_string]& paths)
|
||||
CResult[vector[CFileInfo]] GetFileInfo(const CFileSelector& select)
|
||||
CStatus CreateDir(const c_string& path, c_bool recursive)
|
||||
CStatus DeleteDir(const c_string& path)
|
||||
CStatus DeleteDirContents(const c_string& path, c_bool missing_dir_ok)
|
||||
CStatus DeleteRootDirContents()
|
||||
CStatus DeleteFile(const c_string& path)
|
||||
CStatus DeleteFiles(const vector[c_string]& paths)
|
||||
CStatus Move(const c_string& src, const c_string& dest)
|
||||
CStatus CopyFile(const c_string& src, const c_string& dest)
|
||||
CResult[shared_ptr[CInputStream]] OpenInputStream(
|
||||
const c_string& path)
|
||||
CResult[shared_ptr[CRandomAccessFile]] OpenInputFile(
|
||||
const c_string& path)
|
||||
CResult[shared_ptr[COutputStream]] OpenOutputStream(
|
||||
const c_string& path, const shared_ptr[const CKeyValueMetadata]&)
|
||||
CResult[shared_ptr[COutputStream]] OpenAppendStream(
|
||||
const c_string& path, const shared_ptr[const CKeyValueMetadata]&)
|
||||
c_bool Equals(const CFileSystem& other)
|
||||
c_bool Equals(shared_ptr[CFileSystem] other)
|
||||
|
||||
CResult[shared_ptr[CFileSystem]] CFileSystemFromUri \
|
||||
"arrow::fs::FileSystemFromUri"(const c_string& uri)
|
||||
CResult[shared_ptr[CFileSystem]] CFileSystemFromUri \
|
||||
"arrow::fs::FileSystemFromUri"(const c_string& uri, c_string* out_path)
|
||||
CResult[shared_ptr[CFileSystem]] CFileSystemFromUriOrPath \
|
||||
"arrow::fs::FileSystemFromUriOrPath"(const c_string& uri,
|
||||
c_string* out_path)
|
||||
|
||||
cdef cppclass CFileSystemGlobalOptions \
|
||||
"arrow::fs::FileSystemGlobalOptions":
|
||||
c_string tls_ca_file_path
|
||||
c_string tls_ca_dir_path
|
||||
|
||||
CStatus CFileSystemsInitialize "arrow::fs::Initialize" \
|
||||
(const CFileSystemGlobalOptions& options)
|
||||
|
||||
cdef cppclass CSubTreeFileSystem \
|
||||
"arrow::fs::SubTreeFileSystem"(CFileSystem):
|
||||
CSubTreeFileSystem(const c_string& base_path,
|
||||
shared_ptr[CFileSystem] base_fs)
|
||||
c_string base_path()
|
||||
shared_ptr[CFileSystem] base_fs()
|
||||
|
||||
ctypedef enum CS3LogLevel "arrow::fs::S3LogLevel":
|
||||
CS3LogLevel_Off "arrow::fs::S3LogLevel::Off"
|
||||
CS3LogLevel_Fatal "arrow::fs::S3LogLevel::Fatal"
|
||||
CS3LogLevel_Error "arrow::fs::S3LogLevel::Error"
|
||||
CS3LogLevel_Warn "arrow::fs::S3LogLevel::Warn"
|
||||
CS3LogLevel_Info "arrow::fs::S3LogLevel::Info"
|
||||
CS3LogLevel_Debug "arrow::fs::S3LogLevel::Debug"
|
||||
CS3LogLevel_Trace "arrow::fs::S3LogLevel::Trace"
|
||||
|
||||
cdef struct CS3GlobalOptions "arrow::fs::S3GlobalOptions":
|
||||
CS3LogLevel log_level
|
||||
int num_event_loop_threads
|
||||
|
||||
cdef cppclass CS3ProxyOptions "arrow::fs::S3ProxyOptions":
|
||||
c_string scheme
|
||||
c_string host
|
||||
int port
|
||||
c_string username
|
||||
c_string password
|
||||
c_bool Equals(const CS3ProxyOptions& other)
|
||||
|
||||
@staticmethod
|
||||
CResult[CS3ProxyOptions] FromUriString "FromUri"(
|
||||
const c_string& uri_string)
|
||||
|
||||
ctypedef enum CS3CredentialsKind "arrow::fs::S3CredentialsKind":
|
||||
CS3CredentialsKind_Anonymous "arrow::fs::S3CredentialsKind::Anonymous"
|
||||
CS3CredentialsKind_Default "arrow::fs::S3CredentialsKind::Default"
|
||||
CS3CredentialsKind_Explicit "arrow::fs::S3CredentialsKind::Explicit"
|
||||
CS3CredentialsKind_Role "arrow::fs::S3CredentialsKind::Role"
|
||||
CS3CredentialsKind_WebIdentity \
|
||||
"arrow::fs::S3CredentialsKind::WebIdentity"
|
||||
|
||||
cdef cppclass CS3RetryStrategy "arrow::fs::S3RetryStrategy":
|
||||
@staticmethod
|
||||
shared_ptr[CS3RetryStrategy] GetAwsDefaultRetryStrategy(int64_t max_attempts)
|
||||
|
||||
@staticmethod
|
||||
shared_ptr[CS3RetryStrategy] GetAwsStandardRetryStrategy(int64_t max_attempts)
|
||||
|
||||
cdef cppclass CS3Options "arrow::fs::S3Options":
|
||||
c_string region
|
||||
double connect_timeout
|
||||
double request_timeout
|
||||
c_string endpoint_override
|
||||
c_string scheme
|
||||
c_bool background_writes
|
||||
c_bool allow_delayed_open
|
||||
c_bool allow_bucket_creation
|
||||
c_bool allow_bucket_deletion
|
||||
c_bool check_directory_existence_before_creation
|
||||
c_bool force_virtual_addressing
|
||||
c_string tls_ca_file_path
|
||||
shared_ptr[const CKeyValueMetadata] default_metadata
|
||||
c_string role_arn
|
||||
c_string session_name
|
||||
c_string external_id
|
||||
int load_frequency
|
||||
CS3ProxyOptions proxy_options
|
||||
CS3CredentialsKind credentials_kind
|
||||
shared_ptr[CS3RetryStrategy] retry_strategy
|
||||
void ConfigureDefaultCredentials()
|
||||
void ConfigureAccessKey(const c_string& access_key,
|
||||
const c_string& secret_key,
|
||||
const c_string& session_token)
|
||||
c_string GetAccessKey()
|
||||
c_string GetSecretKey()
|
||||
c_string GetSessionToken()
|
||||
c_bool Equals(const CS3Options& other)
|
||||
|
||||
@staticmethod
|
||||
CS3Options Defaults()
|
||||
|
||||
@staticmethod
|
||||
CS3Options Anonymous()
|
||||
|
||||
@staticmethod
|
||||
CS3Options FromAccessKey(const c_string& access_key,
|
||||
const c_string& secret_key,
|
||||
const c_string& session_token)
|
||||
|
||||
@staticmethod
|
||||
CS3Options FromAssumeRole(const c_string& role_arn,
|
||||
const c_string& session_name,
|
||||
const c_string& external_id,
|
||||
const int load_frequency)
|
||||
|
||||
cdef cppclass CS3FileSystem "arrow::fs::S3FileSystem"(CFileSystem):
|
||||
@staticmethod
|
||||
CResult[shared_ptr[CS3FileSystem]] Make(const CS3Options& options)
|
||||
CS3Options options()
|
||||
c_string region()
|
||||
|
||||
cdef CStatus CInitializeS3 "arrow::fs::InitializeS3"(
|
||||
const CS3GlobalOptions& options)
|
||||
cdef CStatus CEnsureS3Initialized "arrow::fs::EnsureS3Initialized"()
|
||||
cdef CStatus CFinalizeS3 "arrow::fs::FinalizeS3"()
|
||||
cdef CStatus CEnsureS3Finalized "arrow::fs::EnsureS3Finalized"()
|
||||
|
||||
cdef CResult[c_string] ResolveS3BucketRegion(const c_string& bucket)
|
||||
|
||||
cdef cppclass CGcsCredentials "arrow::fs::GcsCredentials":
|
||||
c_bool anonymous()
|
||||
CTimePoint expiration()
|
||||
c_string access_token()
|
||||
c_string target_service_account()
|
||||
|
||||
cdef cppclass CGcsOptions "arrow::fs::GcsOptions":
|
||||
CGcsCredentials credentials
|
||||
c_string endpoint_override
|
||||
c_string scheme
|
||||
c_string default_bucket_location
|
||||
optional[c_string] project_id
|
||||
optional[double] retry_limit_seconds
|
||||
shared_ptr[const CKeyValueMetadata] default_metadata
|
||||
c_bool Equals(const CS3Options& other)
|
||||
|
||||
@staticmethod
|
||||
CGcsOptions Defaults()
|
||||
|
||||
@staticmethod
|
||||
CGcsOptions Anonymous()
|
||||
|
||||
@staticmethod
|
||||
CGcsOptions FromAccessToken(const c_string& access_token,
|
||||
CTimePoint expiration)
|
||||
|
||||
@staticmethod
|
||||
CGcsOptions FromImpersonatedServiceAccount(const CGcsCredentials& base_credentials,
|
||||
c_string& target_service_account)
|
||||
|
||||
cdef cppclass CGcsFileSystem "arrow::fs::GcsFileSystem":
|
||||
@staticmethod
|
||||
CResult[shared_ptr[CGcsFileSystem]] Make(const CGcsOptions& options)
|
||||
CGcsOptions options()
|
||||
|
||||
cdef cppclass CAzureOptions "arrow::fs::AzureOptions":
|
||||
c_string account_name
|
||||
c_string blob_storage_authority
|
||||
c_string dfs_storage_authority
|
||||
c_string blob_storage_scheme
|
||||
c_string dfs_storage_scheme
|
||||
|
||||
c_bool Equals(const CAzureOptions& other)
|
||||
CStatus ConfigureDefaultCredential()
|
||||
CStatus ConfigureAccountKeyCredential(c_string account_key)
|
||||
CStatus ConfigureSASCredential(c_string sas_token)
|
||||
CStatus ConfigureManagedIdentityCredential(c_string client_id)
|
||||
CStatus ConfigureClientSecretCredential(c_string tenant_id,
|
||||
c_string client_id,
|
||||
c_string client_secret)
|
||||
|
||||
cdef cppclass CAzureFileSystem "arrow::fs::AzureFileSystem":
|
||||
@staticmethod
|
||||
CResult[shared_ptr[CAzureFileSystem]] Make(const CAzureOptions& options)
|
||||
CAzureOptions options()
|
||||
|
||||
cdef cppclass CHdfsOptions "arrow::fs::HdfsOptions":
|
||||
HdfsConnectionConfig connection_config
|
||||
int32_t buffer_size
|
||||
int16_t replication
|
||||
int64_t default_block_size
|
||||
|
||||
@staticmethod
|
||||
CResult[CHdfsOptions] FromUriString "FromUri"(
|
||||
const c_string& uri_string)
|
||||
void ConfigureEndPoint(c_string host, int port)
|
||||
void ConfigureDriver(c_bool use_hdfs3)
|
||||
void ConfigureReplication(int16_t replication)
|
||||
void ConfigureUser(c_string user_name)
|
||||
void ConfigureBufferSize(int32_t buffer_size)
|
||||
void ConfigureBlockSize(int64_t default_block_size)
|
||||
void ConfigureKerberosTicketCachePath(c_string path)
|
||||
void ConfigureExtraConf(c_string key, c_string value)
|
||||
|
||||
cdef cppclass CHadoopFileSystem "arrow::fs::HadoopFileSystem"(CFileSystem):
|
||||
@staticmethod
|
||||
CResult[shared_ptr[CHadoopFileSystem]] Make(
|
||||
const CHdfsOptions& options)
|
||||
CHdfsOptions options()
|
||||
|
||||
cdef cppclass CMockFileSystem "arrow::fs::internal::MockFileSystem"(
|
||||
CFileSystem):
|
||||
CMockFileSystem(CTimePoint current_time)
|
||||
|
||||
CStatus CCopyFiles "arrow::fs::CopyFiles"(
|
||||
const vector[CFileLocator]& sources,
|
||||
const vector[CFileLocator]& destinations,
|
||||
const CIOContext& io_context,
|
||||
int64_t chunk_size, c_bool use_threads)
|
||||
CStatus CCopyFilesWithSelector "arrow::fs::CopyFiles"(
|
||||
const shared_ptr[CFileSystem]& source_fs,
|
||||
const CFileSelector& source_sel,
|
||||
const shared_ptr[CFileSystem]& destination_fs,
|
||||
const c_string& destination_base_dir,
|
||||
const CIOContext& io_context,
|
||||
int64_t chunk_size, c_bool use_threads)
|
||||
|
||||
|
||||
# Callbacks for implementing Python filesystems
|
||||
# Use typedef to emulate syntax for std::function<void(..)>
|
||||
ctypedef void CallbackGetTypeName(object, c_string*)
|
||||
ctypedef c_bool CallbackEquals(object, const CFileSystem&)
|
||||
|
||||
ctypedef void CallbackGetFileInfo(object, const c_string&, CFileInfo*)
|
||||
ctypedef void CallbackGetFileInfoVector(object, const vector[c_string]&,
|
||||
vector[CFileInfo]*)
|
||||
ctypedef void CallbackGetFileInfoSelector(object, const CFileSelector&,
|
||||
vector[CFileInfo]*)
|
||||
ctypedef void CallbackCreateDir(object, const c_string&, c_bool)
|
||||
ctypedef void CallbackDeleteDir(object, const c_string&)
|
||||
ctypedef void CallbackDeleteDirContents(object, const c_string&, c_bool)
|
||||
ctypedef void CallbackDeleteRootDirContents(object)
|
||||
ctypedef void CallbackDeleteFile(object, const c_string&)
|
||||
ctypedef void CallbackMove(object, const c_string&, const c_string&)
|
||||
ctypedef void CallbackCopyFile(object, const c_string&, const c_string&)
|
||||
|
||||
ctypedef void CallbackOpenInputStream(object, const c_string&,
|
||||
shared_ptr[CInputStream]*)
|
||||
ctypedef void CallbackOpenInputFile(object, const c_string&,
|
||||
shared_ptr[CRandomAccessFile]*)
|
||||
ctypedef void CallbackOpenOutputStream(
|
||||
object, const c_string&, const shared_ptr[const CKeyValueMetadata]&,
|
||||
shared_ptr[COutputStream]*)
|
||||
ctypedef void CallbackNormalizePath(object, const c_string&, c_string*)
|
||||
|
||||
cdef extern from "arrow/python/filesystem.h" namespace "arrow::py::fs" nogil:
|
||||
|
||||
cdef cppclass CPyFileSystemVtable "arrow::py::fs::PyFileSystemVtable":
|
||||
PyFileSystemVtable()
|
||||
function[CallbackGetTypeName] get_type_name
|
||||
function[CallbackEquals] equals
|
||||
function[CallbackGetFileInfo] get_file_info
|
||||
function[CallbackGetFileInfoVector] get_file_info_vector
|
||||
function[CallbackGetFileInfoSelector] get_file_info_selector
|
||||
function[CallbackCreateDir] create_dir
|
||||
function[CallbackDeleteDir] delete_dir
|
||||
function[CallbackDeleteDirContents] delete_dir_contents
|
||||
function[CallbackDeleteRootDirContents] delete_root_dir_contents
|
||||
function[CallbackDeleteFile] delete_file
|
||||
function[CallbackMove] move
|
||||
function[CallbackCopyFile] copy_file
|
||||
function[CallbackOpenInputStream] open_input_stream
|
||||
function[CallbackOpenInputFile] open_input_file
|
||||
function[CallbackOpenOutputStream] open_output_stream
|
||||
function[CallbackOpenOutputStream] open_append_stream
|
||||
function[CallbackNormalizePath] normalize_path
|
||||
|
||||
cdef cppclass CPyFileSystem "arrow::py::fs::PyFileSystem":
|
||||
@staticmethod
|
||||
shared_ptr[CPyFileSystem] Make(object handler,
|
||||
CPyFileSystemVtable vtable)
|
||||
|
||||
PyObject* handler()
|
||||
Reference in New Issue
Block a user