Slim numerical data compression  1.0
bitstream.h
Go to the documentation of this file.
1 // -*- mode: c++; -*-
2 
5 
6 // Copyright (C) 2008, 2009 Joseph Fowler
7 //
8 // This file is part of slim, a compression package for science data.
9 //
10 // Slim is free software: you can redistribute it and/or modify
11 // it under the terms of the GNU General Public License as published by
12 // the Free Software Foundation, either version 3 of the License, or
13 // (at your option) any later version.
14 //
15 // Slim is distributed in the hope that it will be useful,
16 // but WITHOUT ANY WARRANTY; without even the implied warranty of
17 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 // GNU General Public License for more details.
19 //
20 // You should have received a copy of the GNU General Public License
21 // along with slim. If not, see <http://www.gnu.org/licenses/>.
22 
23 #ifndef SLIM_BITSTREAM_H
24 #define SLIM_BITSTREAM_H
25 
26 #include <stdint.h>
27 #include <cstdio>
28 
29 #ifdef HAVE_LIBZZIP
30 #include <zzip/zzip.h>
31 #endif
32 
33 #ifdef HAVE_LIBLZ4
34 #include <lz4frame.h>
35 #endif
36 
37 #include "bit_constants.h"
38 
39 using namespace std;
40 
41 class bitstream {
42 public:
43  bitstream();
44  bitstream(const char *filename,
45  int buffersize=DEFAULT_IOBUFFER_SIZE);
46  virtual ~bitstream();
47 
48  virtual void close() = 0;
49  virtual bool is_open() const = 0;
50  virtual void setupstream();
51  virtual void windup()=0;
52  virtual int get_bytes_used();
53  int get_bitptr();
54  virtual void print() const = 0;
55 
56 protected:
57  static const int Bits_per_word = 8*sizeof(Word_t);
58 
59  size_t bufsize;
60  size_t buf_used;
61  Byte_t *buffer_base;
62  Byte_t *beyondbuffer;
63  union {
64  Byte_t *Bptr;
65  Word_t *Dptr;
66  } buffptr;
67  int bitptr;
70 
71 public:
72  enum {DEFAULT_IOBUFFER_SIZE=1024*1024};
73  enum {MAX_BITSTREAM_BUFSIZE=16*1024*1024};
74 };
75 
76 
77 
78 class obitstream : public bitstream {
79 private:
80  // No private attributes.
81 
82 protected:
83  FILE *fp;
84 
85 public:
86  obitstream(FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE);
87  obitstream(const char *filename,
88  int buffersize=DEFAULT_IOBUFFER_SIZE);
89  ~obitstream();
90 
91  void writebits(uint32_t data, int nbits);
92  void writestring(const char *str, bool write_trailing_null=false);
93  template <typename T> void writeword(const T data);
94  void write_unary(unsigned int value);
95  virtual void print() const;
96  virtual void close();
97  virtual bool is_open() const;
98  void windup();
99  void flush(bool flush_trailing_bits);
100 };
101 
102 
103 
104 
105 class ibitstream : public bitstream {
106 private:
107  // No private attributes (unless debugging).
108 #ifdef DEBUG_READBITS
109  int c1,c2, c3, c4, c5, c6;
110  void print_debug();
111 #endif
112 
113 protected:
114 #ifdef HAVE_LIBZZIP
115  ZZIP_FILE *zfp;
116 #endif
117  FILE *fp;
118 #ifdef HAVE_LIBLZ4
119  LZ4F_decompressionContext_t lz4_ctx;
120  Byte_t *lz4_buffer;
121  bool using_lz4;
122 #endif
123 
124 public:
125  ibitstream(FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE);
126 #ifdef HAVE_LIBZZIP
127  ibitstream(ZZIP_FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE);
128 #endif
129  ibitstream(const char *filename,
130  int buffersize=DEFAULT_IOBUFFER_SIZE);
131  ibitstream(int fd, int buffersize=DEFAULT_IOBUFFER_SIZE);
132  ~ibitstream();
133 
134  virtual void close();
135  virtual bool is_open() const;
136  void setupstream();
137  void windup();
138  virtual void print() const;
139  virtual int get_bytes_used();
140  Word_t readbits(int nbits);
141  int32_t readbits_int(int nbits);
142  Word_t read_unary();
143  int readstring(char *s, int count=-1);
144  //int get_bits_used() { return bitptr + Bits_per_word*buf_used;}
145 
146 private:
147  void next_word();
148  int fill();
149 
150  Word_t partial_word;
151  int partial_word_bitptr;
152 };
153 
154 
155 
157 // Inline functions
159 
166 static inline unsigned int bit_size(int32_t i) {
167  if (i < 0)
168  i = (-i)-1; // Convert negative int to non-negatives of same size.
169 
170  if (i>lowestNset32bits[15]) {
171  if (i>lowestNset32bits[23]) {
172  if (i>lowestNset32bits[27]) {
173  if (i>lowestNset32bits[29]) {
174  if (i>lowestNset32bits[30]) {
175  return 32;
176  } else {
177  return 31;
178  }
179  } else {
180  if (i>lowestNset32bits[28]) {
181  return 30;
182  } else {
183  return 29;
184  }
185  }
186  } else {
187  if (i>lowestNset32bits[25]) {
188  if (i>lowestNset32bits[26]) {
189  return 28;
190  } else {
191  return 27;
192  }
193  } else {
194  if (i>lowestNset32bits[24]) {
195  return 26;
196  } else {
197  return 25;
198  }
199  }
200  }
201  } else {
202  if (i>lowestNset32bits[19]) {
203  if (i>lowestNset32bits[21]) {
204  if (i>lowestNset32bits[22]) {
205  return 24;
206  } else {
207  return 23;
208  }
209  } else {
210  if (i>lowestNset32bits[20]) {
211  return 22;
212  } else {
213  return 21;
214  }
215  }
216  } else {
217  if (i>lowestNset32bits[17]) {
218  if (i>lowestNset32bits[18]) {
219  return 20;
220  } else {
221  return 19;
222  }
223  } else {
224  if (i>lowestNset32bits[16]) {
225  return 18;
226  } else {
227  return 17;
228  }
229  }
230  }
231  }
232  } else {
233  if (i>lowestNset32bits[7]) {
234  if (i>lowestNset32bits[11]) {
235  if (i>lowestNset32bits[13]) {
236  if (i>lowestNset32bits[14]) {
237  return 16;
238  } else {
239  return 15;
240  }
241  } else {
242  if (i>lowestNset32bits[12]) {
243  return 14;
244  } else {
245  return 13;
246  }
247  }
248  } else {
249  if (i>lowestNset32bits[9]) {
250  if (i>lowestNset32bits[10]) {
251  return 12;
252  } else {
253  return 11;
254  }
255  } else {
256  if (i>lowestNset32bits[8]) {
257  return 10;
258  } else {
259  return 9;
260  }
261  }
262  }
263  } else {
264  if (i>lowestNset32bits[3]) {
265  if (i>lowestNset32bits[5]) {
266  if (i>lowestNset32bits[6]) {
267  return 8;
268  } else {
269  return 7;
270  }
271  } else {
272  if (i>lowestNset32bits[4]) {
273  return 6;
274  } else {
275  return 5;
276  }
277  }
278  } else {
279  if (i>lowestNset32bits[1]) {
280  if (i>lowestNset32bits[2]) {
281  return 4;
282  } else {
283  return 3;
284  }
285  } else {
286  if (i>lowestNset32bits[0]) {
287  return 2;
288  } else {
289  return 1;
290  }
291  }
292  }
293  }
294  }
295 }
296 
297 
298 
303 static inline unsigned int bit_size(unsigned int u) {
304  for (int bs=1; bs<=32; bs++)
305  if (u == (u&lowestNset32bits[bs]))
306  return bs;
307  throw "Bit size (unsigned int) fails!";
308 }
309 
310 
311 #endif // #ifdef SLIM_BITSTREAM_H
static unsigned int bit_size(int32_t i)
Find size (on [0,32]) of the smallest # that can hold the integer i.
Definition: bitstream.h:166
Bit stream base class.
Definition: bitstream.h:41
size_t bufsize
Size of I/O buffer (bytes)
Definition: bitstream.h:59
Byte_t * Bptr
Pointer to the current word (as Byte_t *).
Definition: bitstream.h:64
Byte_t * buffer_base
Pointer to the buffer.
Definition: bitstream.h:61
size_t buf_used
Definition: bitstream.h:60
int bitptr
Pointer to the current bits.
Definition: bitstream.h:67
virtual void print() const =0
Print properties of stream (pure virt)
Word_t * Dptr
Pointer to the current word (as Word_t *).
Definition: bitstream.h:65
Byte_t * beyondbuffer
Pointer just beyond buffer (convenience).
Definition: bitstream.h:62
Input bit stream.
Definition: bitstream.h:105
Output bit stream.
Definition: bitstream.h:78
FILE * fp
The I/O stream.
Definition: bitstream.h:83