Loading...
Searching...
No Matches
ListOffsetArray.h
Go to the documentation of this file.
1// BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
2
3#ifndef AWKWARD_LISTOFFSETARRAY_H_
4#define AWKWARD_LISTOFFSETARRAY_H_
5
6#include <memory>
7
8#include "awkward/common.h"
9#include "awkward/Index.h"
10#include "awkward/Identities.h"
11#include "awkward/Content.h"
12
13namespace awkward {
18 public:
21 ListOffsetForm(bool has_identities,
22 const util::Parameters& parameters,
23 const FormKey& form_key,
24 Index::Form offsets,
25 const FormPtr& content);
26
28 offsets() const;
29
30 const FormPtr
31 content() const;
32
33 const TypePtr
34 type(const util::TypeStrs& typestrs) const override;
35
36 void
37 tojson_part(ToJson& builder, bool verbose) const override;
38
39 const FormPtr
40 shallow_copy() const override;
41
42 const FormPtr
43 with_form_key(const FormKey& form_key) const override;
44
45 const std::string
46 purelist_parameter(const std::string& key) const override;
47
48 bool
49 purelist_isregular() const override;
50
51 int64_t
52 purelist_depth() const override;
53
54 bool
55 dimension_optiontype() const override;
56
57 const std::pair<int64_t, int64_t>
58 minmax_depth() const override;
59
60 const std::pair<bool, int64_t>
61 branch_depth() const override;
62
63 int64_t
64 numfields() const override;
65
66 int64_t
67 fieldindex(const std::string& key) const override;
68
69 const std::string
70 key(int64_t fieldindex) const override;
71
72 bool
73 haskey(const std::string& key) const override;
74
75 const std::vector<std::string>
76 keys() const override;
77
78 bool
79 istuple() const override;
80
81 bool
82 equal(const FormPtr& other,
83 bool check_identities,
84 bool check_parameters,
85 bool check_form_key,
86 bool compatibility_check) const override;
87
88 const FormPtr
89 getitem_field(const std::string& key) const override;
90
91 const FormPtr
92 getitem_fields(const std::vector<std::string>& keys) const override;
93
94 private:
95 Index::Form offsets_;
96 const FormPtr content_;
97 };
98
110 template <typename T>
111 class
112#ifdef AWKWARD_LISTOFFSETARRAY_NO_EXTERN_TEMPLATE
114#endif
115 ListOffsetArrayOf: public Content {
116 public:
132 ListOffsetArrayOf<T>(const IdentitiesPtr& identities,
133 const util::Parameters& parameters,
134 const IndexOf<T>& offsets,
135 const ContentPtr& content);
136
142 const IndexOf<T>
143 offsets() const;
144
150 const ContentPtr
151 content() const;
152
158 const IndexOf<T>
159 starts() const;
160
166 const IndexOf<T>
167 stops() const;
168
177 Index64
178 compact_offsets64(bool start_at_zero) const;
179
185 const ContentPtr
186 broadcast_tooffsets64(const Index64& offsets) const;
187
190 const ContentPtr
192
200 const ContentPtr
201 toListOffsetArray64(bool start_at_zero) const;
202
205 const std::string
206 classname() const override;
207
208 void
209 setidentities() override;
210
211 void
212 setidentities(const IdentitiesPtr& identities) override;
213
214 const TypePtr
215 type(const util::TypeStrs& typestrs) const override;
216
217 const FormPtr
218 form(bool materialize) const override;
219
221 kernels() const override;
222
223 void
224 caches(std::vector<ArrayCachePtr>& out) const override;
225
226 const std::string
227 tostring_part(const std::string& indent,
228 const std::string& pre,
229 const std::string& post) const override;
230
231 void
232 tojson_part(ToJson& builder, bool include_beginendlist) const override;
233
234 void
235 nbytes_part(std::map<size_t, int64_t>& largest) const override;
236
240 int64_t
241 length() const override;
242
243 const ContentPtr
244 shallow_copy() const override;
245
246 const ContentPtr
247 deep_copy(bool copyarrays,
248 bool copyindexes,
249 bool copyidentities) const override;
250
251 void
252 check_for_iteration() const override;
253
254 const ContentPtr
255 getitem_nothing() const override;
256
257 const ContentPtr
258 getitem_at(int64_t at) const override;
259
260 const ContentPtr
261 getitem_at_nowrap(int64_t at) const override;
262
263 const ContentPtr
264 getitem_range(int64_t start, int64_t stop) const override;
265
266 const ContentPtr
267 getitem_range_nowrap(int64_t start, int64_t stop) const override;
268
269 const ContentPtr
270 getitem_field(const std::string& key) const override;
271
272 const ContentPtr
273 getitem_field(const std::string& key,
274 const Slice& only_fields) const override;
275
276 const ContentPtr
277 getitem_fields(const std::vector<std::string>& keys) const override;
278
279 const ContentPtr
280 getitem_fields(const std::vector<std::string>& keys,
281 const Slice& only_fields) const override;
282
283 const ContentPtr
284 getitem_next_jagged(const Index64& slicestarts,
285 const Index64& slicestops,
286 const SliceItemPtr& slicecontent,
287 const Slice& tail) const override;
288
289 const ContentPtr
290 carry(const Index64& carry, bool allow_lazy) const override;
291
292 int64_t
293 purelist_depth() const override;
294
295 const std::pair<int64_t, int64_t>
296 minmax_depth() const override;
297
298 const std::pair<bool, int64_t>
299 branch_depth() const override;
300
301 int64_t
302 numfields() const override;
303
304 int64_t
305 fieldindex(const std::string& key) const override;
306
307 const std::string
308 key(int64_t fieldindex) const override;
309
310 bool
311 haskey(const std::string& key) const override;
312
313 const std::vector<std::string>
314 keys() const override;
315
316 bool
317 istuple() const override;
318
319 // operations
320 const std::string
321 validityerror(const std::string& path) const override;
322
327 const ContentPtr
328 shallow_simplify() const override;
329
330 const ContentPtr
331 num(int64_t axis, int64_t depth) const override;
332
333 const std::pair<Index64, ContentPtr>
334 offsets_and_flattened(int64_t axis, int64_t depth) const override;
335
336 bool
337 mergeable(const ContentPtr& other, bool mergebool) const override;
338
339 bool
340 referentially_equal(const ContentPtr& other) const override;
341
342 const ContentPtr
343 mergemany(const ContentPtrVec& others) const override;
344
345 const SliceItemPtr
346 asslice() const override;
347
348 const ContentPtr
349 fillna(const ContentPtr& value) const override;
350
351 const ContentPtr
352 rpad(int64_t target, int64_t axis, int64_t depth) const override;
353
354 const ContentPtr
355 rpad_and_clip(int64_t target,
356 int64_t axis,
357 int64_t depth) const override;
358
359 const ContentPtr
360 reduce_next(const Reducer& reducer,
361 int64_t negaxis,
362 const Index64& starts,
363 const Index64& shifts,
364 const Index64& parents,
365 int64_t outlength,
366 bool mask,
367 bool keepdims) const override;
368
369 const ContentPtr
370 sort_next(int64_t negaxis,
371 const Index64& starts,
372 const Index64& parents,
373 int64_t outlength,
374 bool ascending,
375 bool stable) const override;
376
377 const ContentPtr
378 argsort_next(int64_t negaxis,
379 const Index64& starts,
380 const Index64& shifts,
381 const Index64& parents,
382 int64_t outlength,
383 bool ascending,
384 bool stable) const override;
385
386 const ContentPtr
387 localindex(int64_t axis, int64_t depth) const override;
388
389 const ContentPtr
390 combinations(int64_t n,
391 bool replacement,
392 const util::RecordLookupPtr& recordlookup,
393 const util::Parameters& parameters,
394 int64_t axis,
395 int64_t depth) const override;
396
397 const ContentPtr
399 const Slice& tail,
400 const Index64& advanced) const override;
401
402 const ContentPtr
404 const Slice& tail,
405 const Index64& advanced) const override;
406
407 const ContentPtr
409 const Slice& tail,
410 const Index64& advanced) const override;
411
412 const ContentPtr
414 const Slice& tail,
415 const Index64& advanced) const override;
416
417 const ContentPtr
418 getitem_next_jagged(const Index64& slicestarts,
419 const Index64& slicestops,
420 const SliceArray64& slicecontent,
421 const Slice& tail) const override;
422
423 const ContentPtr
424 getitem_next_jagged(const Index64& slicestarts,
425 const Index64& slicestops,
426 const SliceMissing64& slicecontent,
427 const Slice& tail) const override;
428
429 const ContentPtr
430 getitem_next_jagged(const Index64& slicestarts,
431 const Index64& slicestops,
432 const SliceJagged64& slicecontent,
433 const Slice& tail) const override;
434
435 const ContentPtr
436 copy_to(kernel::lib ptr_lib) const override;
437
438 const ContentPtr
439 numbers_to_type(const std::string& name) const override;
440
442 bool
443 is_unique() const override;
444
446 const ContentPtr
447 unique() const override;
448
450 bool
451 is_subrange_equal(const Index64& start, const Index64& stop) const override;
452
453 private:
455 const IndexOf<T> offsets_;
457 const ContentPtr content_;
458 };
459
460#ifndef AWKWARD_LISTOFFSETARRAY_NO_EXTERN_TEMPLATE
461 extern template class ListOffsetArrayOf<int32_t>;
462 extern template class ListOffsetArrayOf<uint32_t>;
463 extern template class ListOffsetArrayOf<int64_t>;
464#endif
465
469}
470
471#endif // AWKWARD_LISTOFFSETARRAY_H_
Abstract superclass of all array node types (flat hierarchy). Any Content can be nested within any ot...
Definition: Content.h:276
Abstract superclass of all array node forms, which expresses the nesting structure without any large ...
Definition: Content.h:39
A contiguous, one-dimensional array of integers used to represent data structures,...
Definition: Index.h:82
Form
Integer type of an Index, used by ListForm, IndexedForm, etc.
Definition: Index.h:34
Represents an array of nested lists that can have different lengths using one index named offsets.
Definition: ListOffsetArray.h:115
int64_t purelist_depth() const override
The list-depth of this array, not counting any contained within a RecordArray.
const ContentPtr argsort_next(int64_t negaxis, const Index64 &starts, const Index64 &shifts, const Index64 &parents, int64_t outlength, bool ascending, bool stable) const override
This array sorted indices.
void caches(std::vector< ArrayCachePtr > &out) const override
Accumulates all the unique #ArrayCache objects from nested #VirtualArray nodes. (Uniqueness is determ...
const ContentPtr getitem_fields(const std::vector< std::string > &keys) const override
This array with the first nested RecordArray replaced by a RecordArray of a given subset of keys.
const ContentPtr getitem_next_jagged(const Index64 &slicestarts, const Index64 &slicestops, const SliceArray64 &slicecontent, const Slice &tail) const override
Internal function that propagates a jagged array (array with irregular-length dimensions) slice throu...
const ContentPtr getitem_at(int64_t at) const override
Returns the element at a given position in the array, handling negative indexing and bounds-checking ...
void setidentities() override
Assign a surrogate index of Identities to this array (in-place).
const ContentPtr toRegularArray() const
Converts this array to a RegularArray if all nested lists have the same size (error otherwise).
const ContentPtr getitem_field(const std::string &key, const Slice &only_fields) const override
const ContentPtr getitem_next(const SliceArray64 &array, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ContentPtr unique() const override
Returns an array where all components are unique.
const ContentPtr sort_next(int64_t negaxis, const Index64 &starts, const Index64 &parents, int64_t outlength, bool ascending, bool stable) const override
This array sorted.
const IndexOf< T > stops() const
Stopping positions of each nested list, similar to ListArray::stops, but derived from offsets.
void tojson_part(ToJson &builder, bool include_beginendlist) const override
Internal function to produce a JSON representation one node at a time.
const ContentPtr getitem_fields(const std::vector< std::string > &keys, const Slice &only_fields) const override
int64_t length() const override
The number of elements in the array.
const ContentPtr deep_copy(bool copyarrays, bool copyindexes, bool copyidentities) const override
Copies this node and all nodes hierarchically nested within it, optionally copying the associated arr...
const std::string validityerror(const std::string &path) const override
Returns an error message if this array is invalid; otherwise, returns an empty string.
const ContentPtr getitem_next(const SliceAt &at, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
bool istuple() const override
Returns true if the outermost RecordArray is a tuple.
const IndexOf< T > starts() const
Starting positions of each nested list, similar to ListArray::starts, but derived from offsets.
const ContentPtr getitem_nothing() const override
Internal function to get an empty slice (with the correct type).
kernel::lib kernels() const override
Returns the kernel library enum for all nested ptr_lib within the array's tree structure....
const ContentPtr carry(const Index64 &carry, bool allow_lazy) const override
Returns an array of the same type with elements filtered, rearranged, and possibly duplicated by the ...
const ContentPtr combinations(int64_t n, bool replacement, const util::RecordLookupPtr &recordlookup, const util::Parameters &parameters, int64_t axis, int64_t depth) const override
Tuples or records of all n-tuple combinations of list items at some axis depth.
void setidentities(const IdentitiesPtr &identities) override
Assign a specified set of Identities to this array (in-place).
const ContentPtr toListOffsetArray64(bool start_at_zero) const
Returns a ListOffsetArray with 64-bit offsets and possibly starting with offsets[0] = 0; a shallow_co...
const ContentPtr localindex(int64_t axis, int64_t depth) const override
A (possibly nested) array of integers indicating the positions of elements within each nested list.
const ContentPtr getitem_range_nowrap(int64_t start, int64_t stop) const override
Subinterval of this array, without handling negative indexing or bounds-checking.
const std::pair< Index64, ContentPtr > offsets_and_flattened(int64_t axis, int64_t depth) const override
Returns (a) an offsets Index and (b) a flattened version of the array at some axis depth.
int64_t fieldindex(const std::string &key) const override
The position of a tuple or record key name if this array contains a RecordArray.
void check_for_iteration() const override
Performs up-front validity checks on an array so that they don't have to be checked in getitem_at_now...
const std::vector< std::string > keys() const override
A list of RecordArray keys or an empty list if this array does not contain a RecordArray.
const ContentPtr getitem_field(const std::string &key) const override
This array with the first nested RecordArray replaced by the field at key.
const std::pair< bool, int64_t > branch_depth() const override
Returns (a) whether the list-depth of this array "branches," or differs when followed through differe...
const ContentPtr rpad(int64_t target, int64_t axis, int64_t depth) const override
If axis = 0, returns a view of this array padded on the right with None values to have a minimum leng...
const ContentPtr getitem_at_nowrap(int64_t at) const override
Returns the element at a given position in the array, without handling negative indexing or bounds-ch...
void nbytes_part(std::map< size_t, int64_t > &largest) const override
Internal function used to calculate nbytes.
const ContentPtr mergemany(const ContentPtrVec &others) const override
Returns an array with this and the others concatenated (in order, this first, others last).
const std::string key(int64_t fieldindex) const override
The record name associated with a given field index or the tuple index as a string (e....
const ContentPtr shallow_copy() const override
Copies this node without copying any nodes hierarchically nested within it or any array/index/identit...
const std::pair< int64_t, int64_t > minmax_depth() const override
Returns (a) the minimum list-depth and (b) the maximum list-depth of the array, which can differ if t...
bool is_unique() const override
Returns 'true' if all components of the array are unique.
const FormPtr form(bool materialize) const override
Low-level Form describing all the features of this array except the actual data buffers (Index,...
const SliceItemPtr asslice() const override
Converts this array into a SliceItem that can be used in getitem.
const std::string tostring_part(const std::string &indent, const std::string &pre, const std::string &post) const override
Internal function to build an output string for tostring.
const ContentPtr rpad_and_clip(int64_t target, int64_t axis, int64_t depth) const override
If axis = 0, returns a view of this array padded on the right.
const ContentPtr getitem_next(const SliceJagged64 &jagged, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ContentPtr getitem_range(int64_t start, int64_t stop) const override
Subinterval of this array, handling negative indexing and bounds-checking like Python.
bool is_subrange_equal(const Index64 &start, const Index64 &stop) const override
Returns 'true' if subranges are equal.
const ContentPtr getitem_next_jagged(const Index64 &slicestarts, const Index64 &slicestops, const SliceItemPtr &slicecontent, const Slice &tail) const override
Internal function that propagates a jagged array (array with irregular-length dimensions) slice throu...
const ContentPtr fillna(const ContentPtr &value) const override
Returns this array with None values replaced by a given value.
int64_t numfields() const override
The number of fields in the first nested tuple or records or -1 if this array does not contain a Reco...
bool referentially_equal(const ContentPtr &other) const override
Returns true if this array has all the same buffers and parameters as other; false otherwise.
const TypePtr type(const util::TypeStrs &typestrs) const override
High-level Type describing this array.
const ContentPtr num(int64_t axis, int64_t depth) const override
The length of this array (as a NumpyArray scalar) if axis = 0 or the lengths of subarrays (as an arra...
const ContentPtr broadcast_tooffsets64(const Index64 &offsets) const
Moves content elements if necessary to match a given set of offsets and return a ListOffsetArray that...
bool mergeable(const ContentPtr &other, bool mergebool) const override
Returns true if this array can be merged with the other; false otherwise.
Index64 compact_offsets64(bool start_at_zero) const
Returns 64-bit offsets, possibly starting with offsets[0] = 0.
bool haskey(const std::string &key) const override
Returns true if the array contains a RecordArray with the specified key; false otherwise.
const ContentPtr getitem_next_jagged(const Index64 &slicestarts, const Index64 &slicestops, const SliceJagged64 &slicecontent, const Slice &tail) const override
Internal function that propagates a jagged array (array with irregular-length dimensions) slice throu...
const ContentPtr getitem_next_jagged(const Index64 &slicestarts, const Index64 &slicestops, const SliceMissing64 &slicecontent, const Slice &tail) const override
Internal function that propagates a jagged array (array with irregular-length dimensions) slice throu...
const ContentPtr getitem_next(const SliceRange &range, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ContentPtr shallow_simplify() const override
Returns an equivalent array simplified at one level only using simplify_optiontype if an option-type ...
const ContentPtr content() const
Data contained within all nested lists as a contiguous array.
const IndexOf< T > offsets() const
Positions where one nested list stops and the next starts in the content; the offsets must be monoton...
const ContentPtr reduce_next(const Reducer &reducer, int64_t negaxis, const Index64 &starts, const Index64 &shifts, const Index64 &parents, int64_t outlength, bool mask, bool keepdims) const override
This array with one axis removed by applying a Reducer (e.g. "sum", "max", "any", "all).
const ContentPtr numbers_to_type(const std::string &name) const override
Change the leaf types to 'totype'.
const std::string classname() const override
User-friendly name of this class: "ListOffsetArray32", "ListOffsetArrayU32", or "ListOffsetArray64".
const ContentPtr copy_to(kernel::lib ptr_lib) const override
Recursively copies components of the array from main memory to a GPU (if ptr_lib == kernel::lib::cuda...
Form describing ListOffsetArray.
Definition: ListOffsetArray.h:17
int64_t purelist_depth() const override
The list-depth of this array, not counting any contained within a RecordForm.
const FormPtr with_form_key(const FormKey &form_key) const override
Copies this node, adding or replacing a form_key.
bool equal(const FormPtr &other, bool check_identities, bool check_parameters, bool check_form_key, bool compatibility_check) const override
Returns true if this Form is equal to the other Form; false otherwise.
bool istuple() const override
Returns true if the outermost RecordArray is a tuple.
const FormPtr shallow_copy() const override
Copies this node without copying any nodes hierarchically nested within it.
void tojson_part(ToJson &builder, bool verbose) const override
Internal function to produce a JSON representation one node at a time.
const FormPtr getitem_fields(const std::vector< std::string > &keys) const override
Returns the Form that would result from a fields-slice.
int64_t fieldindex(const std::string &key) const override
The position of a tuple or record key name if this array contains a RecordForm.
const std::vector< std::string > keys() const override
A list of RecordArray keys or an empty list if this array does not contain a RecordArray.
const std::pair< bool, int64_t > branch_depth() const override
Returns (a) whether the list-depth of this array "branches," or differs when followed through differe...
const std::string key(int64_t fieldindex) const override
The record name associated with a given field index or the tuple index as a string (e....
const std::pair< int64_t, int64_t > minmax_depth() const override
Returns (a) the minimum list-depth and (b) the maximum list-depth of the array, which can differ if t...
const FormPtr content() const
const FormPtr getitem_field(const std::string &key) const override
Returns the Form that would result from a field-slice.
int64_t numfields() const override
The number of fields in the first nested tuple or records or -1 if this array does not contain a Reco...
const TypePtr type(const util::TypeStrs &typestrs) const override
High-level Type describing this Form.
const std::string purelist_parameter(const std::string &key) const override
The parameter associated with key at the first level that has a non-null value, descending only as de...
bool haskey(const std::string &key) const override
Returns true if the array contains a RecordForm with the specified key; false otherwise.
Index::Form offsets() const
bool purelist_isregular() const override
Returns true if all nested lists down to the first RecordForm are RegularForm nodes; false otherwise.
bool dimension_optiontype() const override
Returns true if this dimension has option-type; false otherwise.
ListOffsetForm(bool has_identities, const util::Parameters &parameters, const FormKey &form_key, Index::Form offsets, const FormPtr &content)
Creates a ListOffsetForm. See ListOffsetArray for documentation.
Abstract class for all reducer algorithms.
Definition: Reducer.h:20
Represents an array of integers in a slice (possibly converted from an array of booleans).
Definition: Slice.h:229
Represents an integer in a tuple of slices passed to __getitem__ in Python.
Definition: Slice.h:58
Represents an array of nested lists, where the content may be SliceArrayOf, SliceMissingOf,...
Definition: Slice.h:515
Represents a SliceArrayOf, SliceMissingOf, or SliceJaggedOf with missing values: None (no equivalent ...
Definition: Slice.h:435
Represents a Python slice object (usual syntax: array[start:stop:step]).
Definition: Slice.h:93
A sequence of SliceItem objects representing a tuple passed to Python's __getitem__.
Definition: Slice.h:585
Definition: json.h:21
#define LIBAWKWARD_EXPORT_SYMBOL
Definition: common.h:45
lib
Definition: kernel-dispatch.h:20
std::map< std::string, std::string > Parameters
Definition: util.h:165
std::shared_ptr< RecordLookup > RecordLookupPtr
Definition: util.h:130
std::map< std::string, std::string > TypeStrs
Definition: util.h:215
Definition: BitMaskedArray.h:15
std::shared_ptr< std::string > FormKey
Definition: Content.h:19
std::vector< std::shared_ptr< Content > > ContentPtrVec
Definition: Content.h:16
std::shared_ptr< SliceItem > SliceItemPtr
Definition: Slice.h:15
std::shared_ptr< Content > ContentPtr
Definition: Content.h:15
std::shared_ptr< Form > FormPtr
Definition: Content.h:18
std::shared_ptr< Type > TypePtr
Definition: Content.h:23
std::shared_ptr< Identities > IdentitiesPtr
Definition: Identities.h:16