dinfo/dump2fsv.C

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/mman.h>

#include <cstdio>
#include <cstring>
#include <cassert>
#include <map>
#include <vector>
#include <algorithm>

unsigned long sze (int fd)
{
  struct stat st;

  fstat (fd, &st);

  return st.st_size;
}

struct my_int {
   int val;

   my_int(int i = 0) : val(i) { };
};

struct lt_string {
  bool operator ()(char * const &a, char * const &b)
  {
     return strcmp (a, b) < 0;
  }
};

std::map<char *, my_int, lt_string> typedef col_map;

struct {
  bool operator () (col_map::value_type *a, col_map::value_type *b)
  {
     return a->second.val > b->second.val;
  }
} lt_pair;

static int count = 0;

static inline void progress (void)
{
   if (!(++count & 0xfffff))
     {
       printf ("\r%d ", count);
       fflush (stdout);
     }
}

int main(int argc, char *argv[])
{
   char dmp_n[200]; sprintf (dmp_n, "data/dump/%s", argv[1]);
   char row_n[200]; sprintf (row_n, "data/row/%s",  argv[1]);
   char col_n[200]; sprintf (col_n, "data/col/%s.txt",  argv[1]);

   int dmp_fd = open (dmp_n, O_RDWR);
   assert (dmp_fd >= 0);

   int dmp_sze = sze (dmp_fd);
   char *dmp = (char *)mmap (0, dmp_sze, PROT_READ|PROT_WRITE, MAP_SHARED, dmp_fd, 0);

   col_map col;

   printf ("   count\n");
   for (char *c = dmp, *b = dmp; c < dmp + dmp_sze; c++)
     {
       if (!*c)
         {
           ++col[b].val;
           b = c + 1;
           progress ();
         }
     }

   printf ("   copy (%d)\n", count);
   std::vector<col_map::value_type *> srt;

   for (col_map::iterator i = col.begin (); i != col.end (); ++i)
     srt.push_back (&*i);

   printf ("   sort\n");
   sort (srt.begin (), srt.end (), lt_pair);

   FILE *col_f = fopen (col_n, "w");
   int c = 0;
   for (std::vector<col_map::value_type *>::iterator i = srt.begin (); i != srt.end(); ++i, ++c)
     {
       if (c < 100)
         printf ("%3d %8d %8d %s\n", c, (*i)->second.val, (*(i+1))->second.val - (*i)->second.val, (*i)->first);

       (*i)->second.val = c;

       fprintf (col_f, "%d\t%s\n", (*i)->second.val, (*i)->first);
     }

   fclose (col_f);
      
   printf ("   xfrm\n");
   int buf[1024];
   int *bufptr = buf;
   int row_fd = open (row_n, O_RDWR|O_CREAT|O_TRUNC, 0666);
   for (char *c = dmp, *b = dmp; c < dmp + dmp_sze; c++)
     {
       if (!*c)
         {
           *bufptr++ = col[b].val;
           if (bufptr > buf + 1023)
             {
               write (row_fd, buf, (char *)bufptr - (char *)buf);
               bufptr = buf;
             }

           b = c + 1;
           progress ();
         }
     }

   write (row_fd, buf, (char *)bufptr - (char *)buf);
   close (row_fd);
}

#if 0
#!/usr/bin/perl

use Carp;

use PApp::SQL;
use BerkeleyDB;

$DEST = "data/tsv";

my @cols = qw(hausnr name vorname zusatz1 zusatz2 zusatz3 vorwahl strasse plz ort branche typ);

local $PApp::SQL::DBH = PApp::SQL::connect_cached "dinfo::", "DBI:mysql:dinfo";

#sql_exec "truncate $_"
#   for qw(branche haus name ort plz row strasse typ vorname vorwahl zusatz1 zusatz2 zusatz3);

for my $col (@cols) {
   print "$col ANALYZE\n";

   my %col;

   open DMP, "<:raw", "data/dump/$col" or die;
   $col{$_}++ while <DMP>;
   close DMP;

   print "$col SORT\n";
   my @sort = sort { $col{$b} <=> $col{$a} } keys %col;

   print "$col RENUMBER\n";
   open COL, ">:raw", "data/col/$col" or die;
   for (0 .. $#sort) {
      print COL "$_\t$sort[$_]\n";
      $col{$sort[$_]} = $_;
   }
   close COL;

   print "$col CLEAR\n";
   @sort = ();

   print "$col WRITE\n";

   open DMP, "<:raw", "data/dump/$col" or die;
   open ROW, ">:raw", "data/row/$col" or die;

   print ROW "$col{$_}\n" while <DMP>;

   close ROW;
   close DMP;
}

__END__
while (<STDIN>) {
   chomp;
   my @data = split /\t/;
   
   $data[3] =~ s/^, //;
   $data[7] =~ s/\.$// && $data[6] =~ s/ Geb$/Geb./;

   $data[6] =~ /^((?:[a-zA-Z][-.\/a-zA-Z0-9]*)?) (?:\s*\+)? \s* ([0-9\ ]+)$/x or do {
      warn "ERR: unparseable telnr. '$data[6]'";
      next;
   };

   $data[6] = $2;
   $data[13] = $1;

   ($data[5] eq substr $data[6], 0, length $data[5])
      and substr $data[6], 0, length $data[5], "";

   if (length $data[6] > 12) {
      warn "ERR: number too long '$data[6]'";
      next;
   }

   for (@cvtid) {
      $data[$_] = ($cache{"$_,$data[$_]"} ||= do {
         my $data = $data[$_];
         my $fh = $fh{$_} ||= do {
            open my $fh, ">", "$DEST/$cvtid{$_}.txt"
               or die "$DEST/$cvtid{$_}.txt: $!";
            $fh;
         };
         my $id = $data eq "" ? 1 : 1 + ++$seq[$_];
         $data =~ s/([\\\x0a\x09])/\\$1/g; $data =~ s/\x00/\\0/g;

         print $fh "$id\t$data\n";

         $id;
      });
   }

   $data[13] < 16 or die "ERR: too many typ's";

   $data[6] =~ s/ //g;
   $data[6] = pack "H*", (substr "$data[6]000000000000", 0, 12) . sprintf "%x%x", 12 - length $data[6], $data[13];
   #warn unpack "H*", $data[6];
   $data[6] =~ s/([\\\x0a\x09])/\\$1/g; $data[6] =~ s/\x00/\\0/g;
   
   #name vorname zusatz1 zusatz2 zusatz3 vorwahl nummer strasse haus plz ort branche 

   print +(join "\t",
            $data[0], $data[1], $data[2], $data[3], $data[4],
            $data[5], $data[6],
            $data[7], $data[8], $data[9], $data[10], $data[12],
          ), "\n";

   $count++ & 4095 or warn time . " $count";

   if ($count == 20_000_000) {
      while (%cache) {
         warn "copying cache to db...\n";
         while (my ($k, $v) = each %cache) {
            $cache2{$k} = delete $cache{$k};
         }
      }
      warn "nuking mem cache...\n";
      undef %cache;
      warn "assigning db cache...\n";
      *cache = \%cache2;
   }
}

#endif

Revision:	1.2
Committed:	Fri Aug 29 20:17:57 2003 UTC (20 years, 8 months ago) by root
Content type:	text/plain
Branch:	MAIN
CVS Tags:	HEAD
Changes since 1.1:	+0 -0 lines
State:	*FILE REMOVED*
Log Message:	* empty log message *
#	Content
1	#include <sys/types.h>
2	#include <sys/stat.h>
3	#include <fcntl.h>
4	#include <unistd.h>
5	#include <sys/mman.h>
6
7	#include <cstdio>
8	#include <cstring>
9	#include <cassert>
10	#include <map>
11	#include <vector>
12	#include <algorithm>
13
14	unsigned long sze (int fd)
15	{
16	struct stat st;
17
18	fstat (fd, &st);
19
20	return st.st_size;
21	}
22
23	struct my_int {
24	int val;
25
26	my_int(int i = 0) : val(i) { };
27	};
28
29	struct lt_string {
30	bool operator ()(char * const &a, char * const &b)
31	{
32	return strcmp (a, b) < 0;
33	}
34	};
35
36	std::map<char *, my_int, lt_string> typedef col_map;
37
38	struct {
39	bool operator () (col_map::value_type a, col_map::value_type b)
40	{
41	return a->second.val > b->second.val;
42	}
43	} lt_pair;
44
45	static int count = 0;
46
47	static inline void progress (void)
48	{
49	if (!(++count & 0xfffff))
50	{
51	printf ("\r%d ", count);
52	fflush (stdout);
53	}
54	}
55
56	int main(int argc, char *argv[])
57	{
58	char dmp_n[200]; sprintf (dmp_n, "data/dump/%s", argv[1]);
59	char row_n[200]; sprintf (row_n, "data/row/%s", argv[1]);
60	char col_n[200]; sprintf (col_n, "data/col/%s.txt", argv[1]);
61
62	int dmp_fd = open (dmp_n, O_RDWR);
63	assert (dmp_fd >= 0);
64
65	int dmp_sze = sze (dmp_fd);
66	char dmp = (char )mmap (0, dmp_sze, PROT_READ\|PROT_WRITE, MAP_SHARED, dmp_fd, 0);
67
68	col_map col;
69
70	printf (" count\n");
71	for (char c = dmp, b = dmp; c < dmp + dmp_sze; c++)
72	{
73	if (!*c)
74	{
75	++col[b].val;
76	b = c + 1;
77	progress ();
78	}
79	}
80
81	printf (" copy (%d)\n", count);
82	std::vector<col_map::value_type *> srt;
83
84	for (col_map::iterator i = col.begin (); i != col.end (); ++i)
85	srt.push_back (&*i);
86
87	printf (" sort\n");
88	sort (srt.begin (), srt.end (), lt_pair);
89
90	FILE *col_f = fopen (col_n, "w");
91	int c = 0;
92	for (std::vector<col_map::value_type *>::iterator i = srt.begin (); i != srt.end(); ++i, ++c)
93	{
94	if (c < 100)
95	printf ("%3d %8d %8d %s\n", c, (i)->second.val, ((i+1))->second.val - (i)->second.val, (i)->first);
96
97	(*i)->second.val = c;
98
99	fprintf (col_f, "%d\t%s\n", (i)->second.val, (i)->first);
100	}
101
102	fclose (col_f);
103
104	printf (" xfrm\n");
105	int buf[1024];
106	int *bufptr = buf;
107	int row_fd = open (row_n, O_RDWR\|O_CREAT\|O_TRUNC, 0666);
108	for (char c = dmp, b = dmp; c < dmp + dmp_sze; c++)
109	{
110	if (!*c)
111	{
112	*bufptr++ = col[b].val;
113	if (bufptr > buf + 1023)
114	{
115	write (row_fd, buf, (char )bufptr - (char )buf);
116	bufptr = buf;
117	}
118
119	b = c + 1;
120	progress ();
121	}
122	}
123
124	write (row_fd, buf, (char )bufptr - (char )buf);
125	close (row_fd);
126	}
127
128	#if 0
129	#!/usr/bin/perl
130
131	use Carp;
132
133	use PApp::SQL;
134	use BerkeleyDB;
135
136	$DEST = "data/tsv";
137
138	my @cols = qw(hausnr name vorname zusatz1 zusatz2 zusatz3 vorwahl strasse plz ort branche typ);
139
140	local $PApp::SQL::DBH = PApp::SQL::connect_cached "dinfo::", "DBI:mysql:dinfo";
141
142	#sql_exec "truncate $_"
143	# for qw(branche haus name ort plz row strasse typ vorname vorwahl zusatz1 zusatz2 zusatz3);
144
145	for my $col (@cols) {
146	print "$col ANALYZE\n";
147
148	my %col;
149
150	open DMP, "<:raw", "data/dump/$col" or die;
151	$col{$_}++ while <DMP>;
152	close DMP;
153
154	print "$col SORT\n";
155	my @sort = sort { $col{$b} <=> $col{$a} } keys %col;
156
157	print "$col RENUMBER\n";
158	open COL, ">:raw", "data/col/$col" or die;
159	for (0 .. $#sort) {
160	print COL "$_\t$sort[$_]\n";
161	$col{$sort[$_]} = $_;
162	}
163	close COL;
164
165	print "$col CLEAR\n";
166	@sort = ();
167
168	print "$col WRITE\n";
169
170	open DMP, "<:raw", "data/dump/$col" or die;
171	open ROW, ">:raw", "data/row/$col" or die;
172
173	print ROW "$col{$_}\n" while <DMP>;
174
175	close ROW;
176	close DMP;
177	}
178
179	__END__
180	while (<STDIN>) {
181	chomp;
182	my @data = split /\t/;
183
184	$data[3] =~ s/^, //;
185	$data[7] =~ s/\.$// && $data[6] =~ s/ Geb$/Geb./;
186
187	$data[6] =~ /^((?:[a-zA-Z][-.\/a-zA-Z0-9])?) (?:\s\+)? \s* ([0-9\ ]+)$/x or do {
188	warn "ERR: unparseable telnr. '$data[6]'";
189	next;
190	};
191
192	$data[6] = $2;
193	$data[13] = $1;
194
195	($data[5] eq substr $data[6], 0, length $data[5])
196	and substr $data[6], 0, length $data[5], "";
197
198	if (length $data[6] > 12) {
199	warn "ERR: number too long '$data[6]'";
200	next;
201	}
202
203	for (@cvtid) {
204	$data[$_] = ($cache{"$_,$data[$_]"} \|\|= do {
205	my $data = $data[$_];
206	my $fh = $fh{$_} \|\|= do {
207	open my $fh, ">", "$DEST/$cvtid{$_}.txt"
208	or die "$DEST/$cvtid{$_}.txt: $!";
209	$fh;
210	};
211	my $id = $data eq "" ? 1 : 1 + ++$seq[$_];
212	$data =~ s/([\\\x0a\x09])/\\$1/g; $data =~ s/\x00/\\0/g;
213
214	print $fh "$id\t$data\n";
215
216	$id;
217	});
218	}
219
220	$data[13] < 16 or die "ERR: too many typ's";
221
222	$data[6] =~ s/ //g;
223	$data[6] = pack "H*", (substr "$data[6]000000000000", 0, 12) . sprintf "%x%x", 12 - length $data[6], $data[13];
224	#warn unpack "H*", $data[6];
225	$data[6] =~ s/([\\\x0a\x09])/\\$1/g; $data[6] =~ s/\x00/\\0/g;
226
227	#name vorname zusatz1 zusatz2 zusatz3 vorwahl nummer strasse haus plz ort branche
228
229	print +(join "\t",
230	$data[0], $data[1], $data[2], $data[3], $data[4],
231	$data[5], $data[6],
232	$data[7], $data[8], $data[9], $data[10], $data[12],
233	), "\n";
234
235	$count++ & 4095 or warn time . " $count";
236
237	if ($count == 20_000_000) {
238	while (%cache) {
239	warn "copying cache to db...\n";
240	while (my ($k, $v) = each %cache) {
241	$cache2{$k} = delete $cache{$k};
242	}
243	}
244	warn "nuking mem cache...\n";
245	undef %cache;
246	warn "assigning db cache...\n";
247	*cache = \%cache2;
248	}
249	}
250
251	#endif
252