=================================================================== RCS file: /cvs/pta/pta_import.pl,v retrieving revision 1.11 retrieving revision 1.23 diff -u -p -r1.11 -r1.23 --- pta/pta_import.pl 2020/11/17 14:13:31 1.11 +++ pta/pta_import.pl 2020/11/26 19:39:43 1.23 @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/perl # Copyright (c) 2020 Freda Bundchen @@ -21,56 +21,29 @@ use Getopt::Std qw(getopts); our ($opt_I); +my %banks = ( + "bbva_usa" => \&import_bbva_usa, + "capital_one_credit" => \&import_capital_one_credit, + "chase_credit" => \&import_chase_credit, + "etrade_ira" => \&import_etrade_ira, + "optum_hsa" => \&import_optum_hsa, + "sparkasse_camt" => \&import_sparkasse_camt, + "wellsfargo" => \&import_wellsfargo, +); + # === SUBROUTINES ===================================================== -sub import_account ($$$$$) { - my ($account_name, $accounts_ref, $bookings_ref, - $compiled_ref, $csv_account) = @_; - my ($account, $amount, $booking, $credit, $date, $debit, - $description); - chomp(my $header = <>); - while (<>) { - next if (/^$/); - chomp; - my $line = $_; - foreach my $regex (@$compiled_ref) { - if ($line =~ /$regex/) { - $account = %$accounts_ref{$regex}; - $booking = %$bookings_ref{$regex}; - last; - } - } - if ($account_name eq "chase_credit") { - ($date, $amount, $debit, $credit, - $description) = - import_chase_credit($account, - $csv_account, $line); - } elsif ($account_name eq "capital_one_credit") { - ($date, $amount, $debit, $credit, - $description) = - import_capital_one_credit($account, - $csv_account, $line); - } elsif ($account_name eq "optum_hsa") { - ($date, $amount, $debit, $credit, - $description) = - import_optum_hsa($account, - $csv_account, $line); - } elsif ($account_name eq "sparkasse_camt") { - ($date, $amount, $debit, $credit, - $description) = - import_sparkasse_camt($account, - $csv_account, $line); - } else { - die "undefined format: $account_name"; - } - unless ($date && $booking && $debit && $credit && - $amount && $description) { - die "import parse error: $line"; - } - $description =~ s/#//g; - print "$date $booking $debit $credit $amount " . - "$description\n"; - } +sub import_bbva_usa ($$$) { + my ($account, $csv_account, $line) = @_; + $line =~ + m#(\d+)/(\d+)/(\d+),(.*?),\S+,(\(?\d+\.\d+\)?),# or + die "parse error: $line"; + my ($month, $day, $year, $description, $amount) = + ($1, $2, $3, $4, $5); + $amount =~ s/\((\d+\.\d+)\)/-$1/; + return "$year$month$day", + get_accounts_by_amount_sign($amount, $account, $csv_account), + $description; } sub import_chase_credit ($$$) { @@ -78,7 +51,7 @@ sub import_chase_credit ($$$) { my ($trans_date, $post_date, $description, $category, $type, $amount) = split /,/, $line; my ($debit, $credit); - $post_date =~ s#(\d+)/(\d+)/(\d+)#$3$1$2#; + $post_date =~ s#(\d+)/(\d+)/(\d+)#$3$1$2#; ($amount, $debit, $credit) = get_accounts_by_amount_sign($amount, $account, $csv_account); @@ -90,13 +63,27 @@ sub import_capital_one_credit ($$$) { my ($trans_date, $post_date, $card_num, $description, $category, $csv_debit, $csv_credit) = split /,/, $line; - $post_date =~ s/(\d+)-(\d+)-(\d+)/$1$2$3/; - my ($amount, $debit, $credit) = + $post_date =~ s/(\d+)-(\d+)-(\d+)/$1$2$3/; + my ($amount, $debit, $credit) = get_accounts_by_csv_col($account, $csv_account, $csv_debit, $csv_credit); return ($post_date, $amount, $debit, $credit, $description); } +sub import_etrade_ira ($$$) { + my ($account, $csv_account, $line) = @_; + my @fields = split /,/, $line; + $fields[0] =~ s#(\d+)/(\d+)/(\d+)#20$3$1$2#; + my ($amount, $debit, $credit) = + get_accounts_by_amount_sign($fields[5], $account, + $csv_account); + if ($fields[2] eq 'EQ') { + $fields[8] .= " [$fields[3]]"; + } + return ($fields[0], $amount, $debit, $credit, + join ' quantity ', $fields[8], $fields[4]); +} + sub import_optum_hsa ($$$) { my ($account, $csv_account, $line) = @_; my ($date, $description, $amount, @@ -125,15 +112,27 @@ sub import_sparkasse_camt ($$$) { (join ' ', $fields[11], $fields[4]); } +sub import_wellsfargo ($$$) { + my ($account, $csv_account, $line) = @_; + $line =~ + m#^"(\d+)/(\d+)/(\d+)","(-?\d+\.\d+)","\*",".*","(.*?)"$# or + die "parse error: $line" ; + my ($month, $day, $year, $amount, $description) = + ($1, $2, $3, $4, $5); + return "$year$month$day", + get_accounts_by_amount_sign($amount, $account, $csv_account), + $description; +} + sub get_accounts_by_amount_sign ($$$) { my ($amount, $account, $csv_account) = @_; my ($debit, $credit); if ($amount <= 0) { $amount = substr $amount, 1; - $credit = $csv_account; + $credit = $csv_account; $debit = $account; } else { - $debit = $csv_account; + $debit = $csv_account; $credit = $account; } return ($amount, $debit, $credit); @@ -161,38 +160,102 @@ sub usage () { # === MAIN PROGRAM ===================================================== -my ($csv_account, $fn, $in, $account_name, %accounts, %bookings, @compiled); getopts 'I:' or usage; -if ($opt_I) { - $account_name = $opt_I; - $fn = "import_" . $account_name . ".txt"; - open $in, $fn or die "$fn: $!"; -} else { +unless ($opt_I) { + warn "The option -I is required."; usage; } +my $account_name = $opt_I; +$banks{$account_name} or die "unknown accountname: $account_name"; + +# Parse the configuration file. +my $fn = "import/" . $account_name . ".txt"; +open my $in, '<', $fn or die "$fn: $!"; +my ($csv_account, $delim, $header, $quote, @compiled, @ignored); while (<$in>) { chomp; next if /^(?:#|$)/; my $line = $_; - if (/^ACCOUNT\s+(\d+)$/) { + if (/^ACCOUNT\s+(\S+)$/) { $csv_account and die "duplicate ACCOUNT line: $1"; $csv_account = $1; next; } - /^(.*)[,;]\s+(\d+)\s+(\S+)$/ or - die "$fn import parse error: $line"; - my ($reg, $account, $booking) = ($1, $2, $3); - if ($account_name eq 'sparkasse_camt') { - $reg =~ s/(?:^|(?<=;))(?:$|(?=;))/"[^"]*"/g; - } else { - $reg =~ s/(?:^|(?<=,))(?:$|(?=,))/[^,]*/g; + if (/^DELIM\s+(\S)$/) { + $delim and die "duplicate DELIM line: $1"; + $delim = $1; + next; } - $reg = qr/$reg/; - push @compiled, $reg; - $bookings{$reg} = $booking; - $accounts{$reg} = $account; + if (/^HEADER\s+(.*)$/) { + $header and die "duplicate HEADER line: $1"; + $header = $1; + next; + } + if (/^QUOTE\s+(\S)$/) { + $quote and die "duplicate QUOTE line: $1"; + $quote = $1; + next; + } + if (/^IGNORE\s+(.*)/) { + push @ignored, qr/$1/; + next; + } + $delim or die "no DELIM line in $fn"; + s/^(.*)$delim\s+(\d+)\s+(\S+)// or + die "$fn import parse error: $line"; + push @compiled, { + re => [map { qr/$_/ } split /$delim/, $1], + ac => $2, + id => $3, + }; } close $in; $csv_account or die "no ACCOUNT line in $fn"; -import_account($account_name, \%accounts, \%bookings, - \@compiled, $csv_account); + +# Parse the CSV file from the bank. +if (@ARGV) { + open STDIN, '<', $ARGV[0] or die "$ARGV[0]: $!"; +} +LINE: while () { + chomp; + next if (/^$/); + foreach my $ignore (@ignored) { + next LINE if /$ignore/; + } + my $line = $_; + if ($header) { + $line eq $header + or die "expected HEADER $header\nbut got $line"; + undef $header; + next; + } + my $copy_line = $line; + my @fields; + if ($quote) { + push @fields, $1 while $copy_line =~ s/$quote([^$quote]*)$quote$delim?//; + } else { + @fields = split /$delim/, $line; + } + my $matches = 0; + my ($account, $booking); + foreach my $selector (@compiled) { + $matches = 1; + for (my $i = 0; $i <= $#{$selector->{re}}; $i++) { + next if $fields[$i] =~ $selector->{re}[$i]; + $matches = 0; + last; + } + if ($matches) { + $account = $selector->{ac}; + $booking = $selector->{id}; + last; + } + } + $matches or die "unmatched CSV line: $line"; + my ($date, $amount, $debit, $credit, $description) = + $banks{$account_name}->($account, $csv_account, $line); + $date && $amount && $debit && $credit && $description + or die "import parse error: $line"; + $description =~ s/#//g; + print "$date $booking $debit $credit $amount $description\n"; +}