You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
86 lines
4.7 KiB
86 lines
4.7 KiB
[package]
|
|
name = "reverse"
|
|
version = "0.5.0"
|
|
authors = ["Avril <flanchan@cumallover.me>"]
|
|
edition = "2018"
|
|
|
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
|
|
|
[features]
|
|
default = ["mapped-file", "threads"]
|
|
|
|
unstable = ["parking_lot?/nightly", "memchr/compiler_builtins", "crossbeam?/nightly", "futures?/unstable", "color-eyre/track-caller"]
|
|
|
|
# Enables parallelising the operation where possible.
|
|
#
|
|
# Cli config options can control this, but in default auto setting the process will decide if parallelisation is worth using on the dimensions of the provided input and system.
|
|
threads = ["dep:rayon", "dep:parking_lot", "dep:crossbeam", "dep:num_cpus", "threads-async"]
|
|
|
|
# Attempt `mmap()`ping the input instead of reading it.
|
|
# If the output can be mapped, it may also be.
|
|
#
|
|
# Useable on unix-family systems. (TODO: Make this dependency target-dependent for unix instead of a manual feature addition like this.)
|
|
mapped-file = ["dep:mapped-file"]
|
|
|
|
# TODO: Make this non-dependent feature of `threads`: use `tokio_uring` tasks for I/O, and pull in `tokio` **instead of** rayon for operations. For now, we'll see if we can get away with single-threaded async for `tokio_uring` usage and `rayon`'s auto parallisation for the chunk partitioning; if not, change this to not be internal.
|
|
#
|
|
# We are using `tokio_uring` & `tokio-stream` to allow parallel queued writes. (TODO: If we end up using (or needing) this feature for partitioning (e.g. we need a controlled IO thread that takes the data gathered by the parallelised partition system or we need a task pool (NOTE: the task pool will execute on tokio_uring's thread only; rayon's thread pool is being used for auto parallelisation currently) to spawn reverse-reader tasks per partition.) then: Remove the use of crossbeam (XXX: and maybe rayon?), and use Tokio runtime for manual parallelised partitioning/line-reversing.)
|
|
threads-async = ["threads", "dep:futures", "dep:tokio-uring", "dep:tokio-stream"]
|
|
|
|
#rayon = ["dep:rayon"]
|
|
#parking_lot = ["dep:parking_lot"]
|
|
|
|
#XXX ["output-lines", "buffer-output", "ignore-output-errors", "ignore-invalid-args"]
|
|
|
|
#TODO: Move these features to Clap runtime arg parse options.
|
|
|
|
#TODO: Add feature 'mapped-file': Attempt to map input file and process data backwards from end if possible.
|
|
#TODO: Add feature 'threads': Use rayon to parallelise line-searching in buffer (or mapped file ^) (partition in half, move partition line to closest {r,}memchr('\n'); repeat N times recursively on each side (parallel) where N is number of logical CPU cores and/or to get the chunks below/in a certain size range.
|
|
|
|
# Output as lines instead of `["2", "1", "0"]`
|
|
#XXX output-lines = []
|
|
|
|
# Print each string escaped
|
|
#XXX output-quoted = []
|
|
|
|
# Buffer output to conserve syscalls, useful for very large inputs (can cause higher memory usage, but generally speeds output up considerably)
|
|
#XXX buffer-output = []
|
|
|
|
# Do not attempt to handle output errors.
|
|
# Disable this if you are writing to a faulty device or expect some output operations to stdout to fail.
|
|
#XXX ignore-output-errors = []
|
|
|
|
# Ignore invalid arguments instead of removing invalid UTF8 characters if they exist in the argument
|
|
#XXX ignore-invalid-args = []
|
|
|
|
# Operate on raw input byte arrays instead of strings; so non-utf8 characters will be preserved in both input and output
|
|
# NOTE: May cause collecting from stdin to be *slightly* slower when enabled; so only enable if you intend to be operating on non-utf8 strings (which is usually unlikely)
|
|
# NOTE: `ignore-invalid-args` will do nothing if this is enabled.
|
|
#XXX byte-strings = []
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
lto = "fat"
|
|
codegen-units = 1
|
|
strip=true
|
|
|
|
[profile.symbols]
|
|
inherits="release"
|
|
strip=false
|
|
|
|
[dependencies]
|
|
bytes = "1.6.0"
|
|
clap = { version = "4.5.4", features = ["derive", "string", "unicode", "wrap_help"] }
|
|
color-eyre = { version = "0.6.3", default-features = false }
|
|
crossbeam = { version = "0.8.4", optional = true, features = ["crossbeam-queue", "crossbeam-channel", "crossbeam-epoch"] }
|
|
#crossbeam-queue = { version = "0.3.11", optional = true }
|
|
futures = { version = "0.3.30", optional = true, default-features = false, features = ["alloc", "std", "async-await"] }
|
|
lazy_static = { version = "1.4.0", features = ["spin"] }
|
|
mapped-file = { version = "0.0.8", features = ["file"], optional = true }
|
|
memchr = "2.7.2"
|
|
num_cpus = { version = "1.16.0", optional = true }
|
|
parking_lot = { version = "0.12.1", optional = true, features = ["arc_lock", "owning_ref", "send_guard"] }
|
|
rayon = { version = "1.10.0", optional = true }
|
|
tokio-stream = { version = "0.1.15", optional = true, features = ["sync", "fs", "tokio-util", "io-util", "signal"] }
|
|
tokio-uring = { version = "0.4.0", optional = true, features = ["bytes"] }
|