t/120-thumb/data/simple.png
t/130-importer/000-load.t
t/130-importer/010-csv.t
+t/130-importer/020-article.t
+t/130-importer/030-product.t
t/900-kwalitee/010-strict-warn.t
t/900-kwalitee/020-checktemplates.t
t/900-kwalitee/030-messages.t
t/cfg/isafile.cfg
t/cfg/t/varinc.cfg
t/data/govhouse.jpg
+t/data/importer/article-simple.csv
t/data/importer/basic.csv
+t/data/importer/product-simple.csv
t/data/known_pod_issues.txt
t/data/t101.jpg
t/t000load.t
use strict;
use Config;
-our $VERSION = "1.005";
+our $VERSION = "1.006";
=head1 NAME
value with C<BSE::Importer::Target::> prepended, so a value of
C<Product> will use the C<BSE::Importer::Target::Product> module.
+=item *
+
+C<update_only> - if true, the profile will only update existing
+records. This may change which fields are required.
+
=back
The source and target module may include their own configuration in
C<callback> - a sub ref to call for messages generated during
processing.
+=item *
+
+C<listen> - a hashref of event handlers.
+
=back
If the profile is invalid, new() with die with a newline terminated
}
$self->{file_path} = \@file_path;
+ $self->{update_only} = $self->cfg_entry('update_only', 0);
my $source_type = $self->cfg_entry("source", "XLS");
$self->{source_class} = "BSE::Importer::Source::$source_type";
opts => \%opts,
);
-
return $self;
}
defined $value && $value =~ /\S/
and push @parents, $value;
}
+ $self->event(row => { entry => \%entry, parents => \@parents });
$self->{target}->row($self, \%entry, \@parents);
};
if ($@) {
$error =~ tr/\n/ /s;
push @{$self->{errors}}, $error;
$self->warn("Error: $error");
+ $self->event(error => { msg => $error });
}
}
and $self->{callback}->($self->{source}->rowid, ": @msg");
}
+=item event()
+
+Called by various parts of the system to report events. These are
+intended for tools.
+
+=cut
+
+sub event {
+ my ($self, $event, $args) = @_;
+
+ if ($self->{listen}{$event}) {
+ $self->{listen}{$event}->($event, $args);
+ }
+}
+
=item find_file()
my $fullname = $imp->find_file($filename)
return $self->{cfg}->entry($self->{section}, $key, $default);
}
+=item update_only
+
+Returns true if only performing updates.
+
+=cut
+
+sub update_only {
+ $_[0]{update_only};
+}
+
1;
=back
use Products;
use OtherParents;
-our $VERSION = "1.002";
+our $VERSION = "1.003";
=head1 NAME
Provides a target for importing BSE articles.
-The import profile must provide a C<title> mapping.
+C<update_only> profiles must provide a mapping for one of C<id> or
+C<linkAlias>.
+
+Non-C<update_only> profiles must provide a mapping for C<title>.
=head1 CONFIGURATION
=item *
C<codes> - set to true to use the configured C<code_field> to update
-existing articles rather than creating new articles.
+existing articles rather than creating new articles. This is forced
+on when the import profile enables C<update_only>.
=item *
my $importer = delete $opts{importer};
+ $self->{use_codes} = $importer->cfg_entry('codes', 0);
my $map = $importer->maps;
- defined $map->{title}
- or die "No title mapping found\n";
+ if ($importer->update_only) {
+ my $def_code;
+ my $found_key = 0;
+ KEYS:
+ for my $key ($self->key_fields) {
+ if ($map->{$key}) {
+ $found_key = 1;
+ $def_code = $key;
+ last KEYS;
+ }
+ }
+ $found_key
+ or die "No key field (", join(",", $self->key_fields),
+ ") mapping found\n";
- $self->{use_codes} = $importer->cfg_entry('codes', 0);
- $self->{code_field} = $importer->cfg_entry("code_field", $self->default_code_field);
+ $self->{code_field} = $importer->cfg_entry("code_field", $def_code);
+ $self->{use_codes} = 1;
+ }
+ else {
+ defined $map->{title}
+ or die "No title mapping found\n";
+
+ $self->{code_field} = $importer->cfg_entry("code_field", $self->default_code_field);
+
+ }
$self->{parent} = $importer->cfg_entry("parent", $self->default_parent);
my ($self, $importer, $entry, $parents) = @_;
$self->xform_entry($importer, $entry);
-
- $entry->{parentid} = $self->_find_parent($importer, $self->{parent}, @$parents);
+
+ if (!$importer->update_only || @$parents) {
+ $entry->{parentid} = $self->_find_parent($importer, $self->{parent}, @$parents);
+ }
+
my $leaf;
if ($self->{use_codes}) {
my $leaf_id = $entry->{$self->{code_field}};
-
- $leaf = $self->find_leaf($leaf_id);
+
+ if ($importer->{update_only}) {
+ $leaf_id =~ /\S/
+ or die "$self->{code_field} blank for update_only profile\n";
+ }
+
+ $leaf = $self->find_leaf($leaf_id, $importer);
}
if ($leaf) {
@{$leaf}{keys %$entry} = values %$entry;
$leaf->save;
- $importer->info("Updated $leaf->{id}: $entry->{title}");
+ $importer->info("Updated $leaf->{id}: ".$leaf->title);
if ($self->{reset_images}) {
$leaf->remove_images($importer->cfg);
$importer->info(" $leaf->{id}: Reset images");
}
}
}
- else {
+ elsif (!$importer->update_only) {
$leaf = $self->make_leaf
(
$importer,
);
$importer->info("Added $leaf->{id}: $entry->{title}");
}
+ else {
+ die "No leaf found for $entry->{$self->{code_field}} for update_only profile\n";
+ }
for my $image_index (1 .. 10) {
my $file = $entry->{"image${image_index}_file"};
$file
}
$self->fill_leaf($importer, $leaf, %$entry);
push @{$self->{leaves}}, $leaf;
+
+ $importer->event(endrow => { leaf => $leaf });
}
=item xform_entry()
sub xform_entry {
my ($self, $importer, $entry) = @_;
- $entry->{title} =~ /\S/
- or die "title blank\n";
-
- $entry->{title} =~ /\n/
- and die "Title may not contain newlines";
- $entry->{summary}
- or $entry->{summary} = $entry->{title};
- $entry->{description}
- or $entry->{description} = $entry->{title};
- $entry->{body}
- or $entry->{body} = $entry->{title};
+ if (exists $entry->{title}) {
+ $entry->{title} =~ /\S/
+ or die "title blank\n";
+
+ $entry->{title} =~ /\n/
+ and die "Title may not contain newlines";
+ }
+ unless ($importer->update_only) {
+ $entry->{summary}
+ or $entry->{summary} = $entry->{title};
+ $entry->{description}
+ or $entry->{description} = $entry->{title};
+ $entry->{body}
+ or $entry->{body} = $entry->{title};
+ }
}
=item children_of()
=cut
sub find_leaf {
- my ($self, $leaf_id) = @_;
+ my ($self, $leaf_id, $importer) = @_;
- $leaf_id =~ tr/A-Za-z0-9_/_/cds;
+ $leaf_id =~ s/\A\s+//;
+ $leaf_id =~ s/\s+\z//;
my ($leaf) = Articles->getBy($self->{code_field}, $leaf_id)
or return;
+ $importer->event(find_leaf => { id => $leaf_id, leaf => $leaf });
+
return $leaf;
}
sub make_leaf {
my ($self, $importer, %entry) = @_;
- return bse_make_article(%entry);
+ my $leaf = bse_make_article(%entry);
+
+ $importer->event(make_leaf => { leaf => $leaf });
+
+ return $leaf;
}
=item fill_leaf()
Return the default code field.
-Overridden by the produ
uct target to return the C<product_code> field.
+Overridden by the product target to return the C<product_code> field.
=cut
return @{$_[0]{parents}}
}
+=item key_fields()
+
+Columns that can act as keys.
+
+=cut
+
+sub key_fields {
+ return qw(id linkAlias);
+}
+
1;
=back
use BSE::TB::ProductOptionValues;
use BSE::TB::PriceTiers;
-our $VERSION = "1.002";
+our $VERSION = "1.003";
=head1 NAME
$self->{reset_prodopts} = $importer->cfg_entry("reset_prodopts", 1);
my $map = $importer->maps;
- defined $map->{retailPrice}
- or die "No retailPrice mapping found\n";
+ unless ($importer->update_only) {
+ defined $map->{retailPrice}
+ or die "No retailPrice mapping found\n";
+ }
$self->{price_tiers} = +{ map { $_->id => $_ } BSE::TB::PriceTiers->all };
$self->SUPER::xform_entry($importer, $entry);
if ($self->{use_codes}) {
- $entry->{product_code} =~ /\S/
- or die "product_code blank with use_codes\n";
+ $entry->{$self->{code_field}} =~ /\S/
+ or die "$self->{code_field} blank with use_codes\n";
}
- $entry->{retailPrice} =~ s/\$//; # in case
- if ($entry->{retailPrice} =~ /\d/) {
- $self->{price_dollar}
- and $entry->{retailPrice} *= 100;
- }
- else {
- $importer->warn("Warning: no price");
- $entry->{retailPrice} = 0;
+ if (exists $entry->{retailPrice}) {
+ $entry->{retailPrice} =~ s/\$//; # in case
+
+ if ($entry->{retailPrice} =~ /\d/) {
+ $self->{price_dollar}
+ and $entry->{retailPrice} *= 100;
+ }
+ else {
+ $importer->warn("Warning: no price");
+ $entry->{retailPrice} = 0;
+ }
}
}
=cut
sub find_leaf {
- my ($self, $leaf_id) = @_;
+ my ($self, $leaf_id, $importer) = @_;
my ($leaf) = Products->getBy($self->{code_field}, $leaf_id)
or return;
+ $importer->event(find_leaf => { id => $leaf_id, leaf => $leaf });
+
if ($self->{reset_prodopts}) {
my @options = $leaf->db_options;
for my $option (@options) {
sub make_leaf {
my ($self, $importer, %entry) = @_;
- return bse_make_product(%entry);
+ my $leaf = bse_make_product(%entry);
+
+ $importer->event(make_leaf => { leaf => $leaf });
+
+ return $leaf;
}
=item fill_leaf()
sub default_code_field { "product_code" }
+=item key_fields
+
+Fields that can act as key fields.
+
+=cut
+
+sub key_fields {
+ my ($class) = @_;
+
+ return ( $class->SUPER::key_fields(), "product_code" );
+}
+
1;
=back
--- /dev/null
+#!perl -w
+use strict;
+use BSE::Test qw(base_url);
+use File::Spec;
+use File::Temp;
+
+use Test::More tests => 5;
+
+BEGIN {
+ unshift @INC, File::Spec->catdir(BSE::Test::base_dir(), "cgi-bin", "modules");
+}
+
+use BSE::Importer;
+use BSE::API qw(bse_init bse_make_article);
+
+my $base_cgi = File::Spec->catdir(BSE::Test::base_dir(), "cgi-bin");
+ok(bse_init($base_cgi), "initialize api")
+ or print "# failed to bse_init in $base_cgi\n";
+
+my $when = time;
+
+my $cfg = BSE::Cfg->new(path => $base_cgi, extra_text => <<CFG);
+[import profile simple$when]
+map_title=1
+source=CSV
+target=Article
+
+[import profile simpleupdate$when]
+map_linkAlias=1
+map_body=2
+source=CSV
+target=Article
+update_only=1
+sep_char=\\t
+CFG
+
+{
+ my @added;
+
+ my $imp = BSE::Importer->new(cfg => $cfg, profile => "simple$when", callback => sub { note @_ });
+ $imp->process("t/data/importer/article-simple.csv");
+ @added = sort { $a->title cmp $b->title } $imp->leaves;
+
+ is(@added, 2, "imported two articles");
+ is($added[0]->title, "test1", "check title of first import");
+ is($added[1]->title, "test2", "check title of second import");
+
+ END {
+ $_->remove($cfg) for @added;
+ }
+}
+
+{
+ my $testa = bse_make_article(cfg => $cfg, title => "test updates",
+ linkAlias => "alias$when");
+
+ my $fh = File::Temp->new;
+ my $filename = $fh->filename;
+ print $fh <<EOS;
+linkAlias\tbody
+"alias$when"\t"This is the body text with multiple lines
+
+Yes, multiple lines with CSV!"
+EOS
+ close $fh;
+ my $imp = BSE::Importer->new(cfg => $cfg, profile => "simpleupdate$when", callback => sub { note @_ });
+ $imp->process($filename);
+ my $testb = Articles->getByPkey($testa->id);
+ like($testb->body, qr/This is the body/, "check the body is updated");
+
+ END {
+ $testa->remove($cfg) if $testa;
+ }
+}
--- /dev/null
+#!perl -w
+use strict;
+use BSE::Test qw(base_url);
+use File::Spec;
+use File::Temp;
+
+use Test::More tests => 7;
+
+BEGIN {
+ unshift @INC, File::Spec->catdir(BSE::Test::base_dir(), "cgi-bin", "modules");
+}
+
+use BSE::Importer;
+use BSE::API qw(bse_init bse_make_product);
+
+my $base_cgi = File::Spec->catdir(BSE::Test::base_dir(), "cgi-bin");
+ok(bse_init($base_cgi), "initialize api")
+ or print "# failed to bse_init in $base_cgi\n";
+
+my $when = time;
+
+my $cfg = BSE::Cfg->new(path => $base_cgi, extra_text => <<CFG);
+[import profile simple$when]
+map_title=1
+map_retailPrice=2
+source=CSV
+price_dollar=1
+
+[import profile simpleupdate$when]
+map_linkAlias=1
+map_body=2
+source=CSV
+target=Product
+update_only=1
+sep_char=\\t
+code_field=linkAlias
+CFG
+
+{
+ my @added;
+
+ my $imp = BSE::Importer->new(cfg => $cfg, profile => "simple$when",
+ callback => sub { note @_ });
+ $imp->process("t/data/importer/product-simple.csv");
+ @added = sort { $a->title cmp $b->title } $imp->leaves;
+
+ is(@added, 2, "imported two products");
+ is($added[0]->title, "test1", "check title of first import");
+ is($added[0]->retailPrice, 1000, "check price of first import");
+ is($added[1]->title, "test2", "check title of second import");
+ is($added[1]->retailPrice, 800, "check price of second import");
+
+ END {
+ $_->remove($cfg) for @added;
+ }
+}
+
+{
+ my $testa = bse_make_product(cfg => $cfg, title => "test updates",
+ linkAlias => "P$when", retailPrice => 500);
+
+ my $fh = File::Temp->new;
+ my $filename = $fh->filename;
+ print $fh <<EOS;
+linkAlias\tbody
+"P$when"\t"This is the body text with multiple lines
+
+Yes, multiple lines with CSV!"
+EOS
+ close $fh;
+ my $imp = BSE::Importer->new(cfg => $cfg, profile => "simpleupdate$when",
+ callback => sub { note @_ });
+ $imp->process($filename);
+ my $testb = Articles->getByPkey($testa->id);
+ like($testb->body, qr/This is the body/, "check the body is updated");
+
+ END {
+ $testa->remove($cfg) if $testa;
+ }
+}
--- /dev/null
+title
+test1
+test2
--- /dev/null
+title,retailPrice
+test1,10.00
+test2,$8.00
projects / bse.git / commitdiff