Add this to your schema.yml

  state:
    id:
    name:          varchar(50)
  city:
    id:
    state_id:
    name:          varchar(50)
 

Create a batch/load_citystates.php

define('SF_ROOT_DIR',    realpath(dirname(__FILE__).'/..'));
define('SF_APP',         'frontend');
define('SF_ENVIRONMENT', 'dev');
define('SF_DEBUG',       true);
 
require_once(SF_ROOT_DIR.DIRECTORY_SEPARATOR.'apps'.DIRECTORY_SEPARATOR.SF_APP.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'config.php');
 
sfContext::getInstance();
 
// initialize database manager
$databaseManager = new sfDatabaseManager();
$databaseManager->initialize();
 
echo 'Loading States - ';
 
$page = file_get_contents('http://www.citytowninfo.com/places');
 
preg_match_all('@<tr><td><a href="http://www.citytowninfo.com/places/[^"]*"><strong>([a-z-A-Z ]*)</strong></a></td><td>[^<]*</td><td><a href="http://www.citytowninfo.com/places/[^/]*/[^"]*">[^<]*</a></td></tr>@i', $page, $states, PREG_SET_ORDER);
 
echo count($states). "\n";
 
for ($i = 0; $i < count($states); $i++)
{
    $state = new State();
    $state->setName($states[$i][1]);
    $state->save();
 
    $page  = file_get_contents('http://www.citytowninfo.com/places/' . str_replace(' ', '-', strtolower($state->getName())) . '/alphabetically/a-d');
    $page .= file_get_contents('http://www.citytowninfo.com/places/' . str_replace(' ', '-', strtolower($state->getName())) . '/alphabetically/e-h');
    $page .= file_get_contents('http://www.citytowninfo.com/places/' . str_replace(' ', '-', strtolower($state->getName())) . '/alphabetically/i-l');
    $page .= file_get_contents('http://www.citytowninfo.com/places/' . str_replace(' ', '-', strtolower($state->getName())) . '/alphabetically/m-p');
    $page .= file_get_contents('http://www.citytowninfo.com/places/' . str_replace(' ', '-', strtolower($state->getName())) . '/alphabetically/q-u');
    $page .= file_get_contents('http://www.citytowninfo.com/places/' . str_replace(' ', '-', strtolower($state->getName())) . '/alphabetically/v-z');
 
    preg_match_all('@<tr><td><a href="/places/[^/]*/[^/]*"><strong>([a-z-A-Z ]*)</strong></a></td><td>[^<]*</td><td>[^<]*</td></tr>@i', $page, $cities, PREG_SET_ORDER);
?>
Getting cities for <?php echo $state->getName(); ?> - <?php echo count($cities); ?>
 
<?php
    for ($j = 0; $j < count($cities); $j++)
    {
        $city = new City();
        $city->setName($cities[$j][1]);
        $city->setStateId($state->getId());
        $city->save();
    }
}
 

I create a script to use complex arguments while runing script from command line.

It also allow to switch SF_ENVIRONMENT, SF_APP and SF_DEBUG directly from command line.

Just create configBatch.php in your project's config directory and paste this :

<?php
define('SF_ROOT_DIR',    realpath(dirname(__file__).'/..'));
 
$argv = array();
for ($i = 1; $i < $_SERVER["argc"]; $i++) {
    if ($_SERVER["argv"][$i]{0} === '-') {
        // argument
        $value =  (
            isset($_SERVER["argv"][$i+1]) 
            && 
            $_SERVER["argv"][$i+1]{0} !== '-'
            ?
            $_SERVER["argv"][$i+1]
            :
            true
        );
 
        if ($_SERVER["argv"][$i]{1} === '-') {
            // long argument
            $argv[substr($_SERVER["argv"][$i], 2)] = $value;
        }
        else {
            foreach (str_split($_SERVER["argv"][$i]) as $arg) {
                if (ereg('[a-zA-Z0-9]', $arg)) {
                    $last_arg   = $arg;
                    $argv[$arg] = true;
                }
            }
            $argv[$last_arg] = $value;
        }
    }
}
 
define('SF_APP',         (isset($argv['app'])?$argv['app']:'www'));
define('SF_ENVIRONMENT', (isset($argv['env'])?$argv['env']:'prod'));
define('SF_DEBUG',       (isset($argv['debug'])?1:0));
 
require_once(SF_ROOT_DIR.DIRECTORY_SEPARATOR.'apps'.DIRECTORY_SEPARATOR.SF_APP.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'config.php');
 

Then, start your batch file with :

<?php
 
require_once(realpath(dirname(__file__).'/..').DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'configBatch.php');
 
// Your script start here
 

Now, you can pass arguments to you script :

php -f /path/to/project/batch/my_batch.php -rf some_value --help --foo bar
 

Here is the generated $argv :

$argv = array (
  'r' => true,
  'f' => 'some_value',
  'help' => true,
  'foo' => 'bar',
);
 

You can also change the SF_ENVIRONMENT with the --env argument, the SF_APP with the --app argument and the debug flag with --debug argumnent

Enjoy

Normally, there is a controller for each environment in your app. However, batch scripts have only one controller, and hence you have to set the environment manually.

This code allows you to set the environment to 'dev' while developing, and to 'prod' when your batch script has to run on the production server. The clever thing here is that this is done transparently on the production server (whoever calls your scripts do not have to make a choice of environment).

Create a file batchConfig.php (eg. in config folder):

$env = (((count($argv) >= 2) && ($argv[1] == 'dev')) ? 'dev' : 'prod');
 
define('SF_ROOT_DIR',    realpath(dirname(__file__).'/..'));
define('SF_APP',         'frontend');
define('SF_ENVIRONMENT', $env);
define('SF_DEBUG',       true);
 
require_once(SF_ROOT_DIR.DIRECTORY_SEPARATOR.'apps'.DIRECTORY_SEPARATOR.SF_APP.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'config.php');
 
// Initialize database manager.
$databaseManager = new sfDatabaseManager();
$databaseManager->initialize();
 

(The first line does all the work.) In all your batch files, just include it with:

require_once('config/batchConfig.php');
 

and leave out all the usual definitions and database connection code.

In dev, just call:

php myscript.php dev
 

and do as usual in prod:

php myscript.php
 

As an added benefit, your batch files will only contain actual business logic.

It's very annoying that the pake tasks @doctrine-build-all@ and @doctrine-build-all-load@ don't work. So I've thought, why don't write a shell script which executes the single steps for me.

Just create a new file in your batch directory. For example 'build' or 'doctrine'

#!/usr/bin/env bash
 
symfony doctrine-drop-db <your app>
symfony doctrine-build-db <your app>
symfony doctrine-build-model <your app>
symfony doctrine-build-sql <your app>
symfony doctrine-insert-sql <your app>
 
php batch/<your load-data-batch-file>.php

It can be a hassle to write a package.yml for a plugin if it contains many files. Fortunately, with the help of the sfFinder class and a bit of programming, you can automate the generation of the <content> part in a batch.

Create a batch/package_generator.php with the following code:

<?php
 
define('SF_ROOT_DIR',    realpath(dirname(__FILE__).'/..'));
define('SF_APP',         'frontend');
define('SF_ENVIRONMENT', 'prod');
define('SF_DEBUG',       false);
 
require_once(SF_ROOT_DIR.DIRECTORY_SEPARATOR.'apps'.DIRECTORY_SEPARATOR.SF_APP.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'config.php');
 
$line          = array();
$lines         = array();
$previous_dirs = array();  
$files         = sfFinder::type('file')->relative()->in('path/to/plugin/files');
 
// Prepare the list of files for nice hierarchy
foreach($files as $file)
{
  $file = str_replace('\\', '/', $file);
  $dirs = explode ('/', $file);
  $line['filename'] = array_pop($dirs);
  $temp_dirs = $dirs;
  foreach($dirs as $dir)
  {
    if($previous_dirs && $dir == $previous_dirs[0])
    {
      array_shift($previous_dirs); 
      array_shift($dirs);
    } 
  }
  foreach($dirs as $dir)
  {
    $line['open'][] = $dir;        
  }
  foreach($previous_dirs as $dir)
  {
    $line['close'][] = $dir;        
  }
  $lines[] = $line;
  $line = array();
  $previous_dirs = $temp_dirs;
}
$blanks = 0;
?>
 
<?php foreach($lines as $line): ?>
<?php if(isset($line['close'])): ?>
<?php foreach($line['close'] as $dir): ?>
<?php echo str_repeat('  ', --$blanks) ?></dir>
<?php endforeach; ?>
<?php endif; ?>
<?php if(isset($line['open'])): ?>
<?php foreach($line['open'] as $dir): $counter++ ?>
<?php echo str_repeat('  ', $blanks++) ?><dir name="<?php echo $dir ?>">
<?php endforeach; ?>
<?php endif; ?>
<?php echo str_repeat('  ', $blanks) ?><file name="<?php echo str_replace('\\', '/', $line['filename']) ?>" role="data"/>
<?php endforeach; ?>

Just run it with

> php batch/package_generator.php

and paste the output in your package.xml. That's all.

I was looking for this type of functionality:

'php batch/mybatch.php app=batch env=dev'

The following allows me to use the same batch scripts for different environments and applications. It also keeps batch scripts lean by only requiring one line.

batch/run_me.php:

require_once('lib/batch.php');
 
//
// begin batch scripting here
//

batch/lib/batch.php:

set_time_limit(0);
 
$app = get_app($argv[1]);
$env = get_env($argv[2]);
$debug = get_debug($argv[3]);
 
//
// initialize symfony
//
 
define('SF_ROOT_DIR',         realpath(dirname(__FILE__).'/../..'));
define('SF_APP',              $app);
define('SF_ENVIRONMENT',      $env);
define('SF_DEBUG',            $debug);
 
require_once(SF_ROOT_DIR.DIRECTORY_SEPARATOR.'apps'.DIRECTORY_SEPARATOR.SF_APP.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'config.php');
 
sfContext::getInstance();
 
//
// initialize doctrine (just an example)
//
 
$connection = sfDoctrine::connection();
 
//
// parse command line args
//
 
function get_app($argv)
{
  preg_match('@app=(\w+)@', $argv, $match);
 
  return isset($match[1]) ? $match[1] : 'batch';
}
 
function get_env($argv)
{
  preg_match('@env=(\w+)@', $argv, $match);
 
  return isset($match[1]) ? $match[1] : 'dev';
}
 
function get_debug($argv)
{
  preg_match('@(true|false)@', $argv, $match);
 
  return isset($match[1]) ? $match[1] : true;
 
}

Sending a large volume of emails using symfony's built-in email support can be problematic because of memory overhead involved in calling the sendEmail action for every email to be sent. In order to overcome this, we will have to use another way. In essence, instead of making multiple symfony mail calls, we will grab the presentation for the email action, then use sfMail directly to send our emails.

In an action

// Get the presentation only for the actual email.
// email/newsletter is the module/action pair that will
// generate the template for the actual email in this example.
$raw_email = $this->getPresentationFor('email', 'newsletter');
 
require_once sfConfig::get('sf_symfony_lib_dir') . '/addon/sfMail/sfMail.class.php';
$mail = new sfMail();
$mail->setMailer('sendmail');
$mail->setCharset('utf-8');
$mail->setContentType('text/html');
 
$mail->setFrom($sender_email, $sender_name);
$mail->setSubject($newsletter_title);
$mail->setBody($raw_email);
 
// To further ensure that we don't choke up on the
// emails, we introduce a throttle. In this example,
// we use 100, which we will use to sleep for a period
// of time for every 100 emails sent.
 
$throttle = 100;
$i = 0;
 
// Now, run a loop through all the email addresses in the mailing list.
 
foreach($email_addrs as $email_addr)
{
    if ($i % $throttle == 0)
    {
        sleep(10); // Here, we sleep for 10 seconds for every $throttle emails sent.
    }
 
    $mail->addAddress($email_addr);
    $mail->prepare();
    $mail->send();
    $mail->clearAllRecipients();
    $i++;
}

That's it! Now, just go ahead and set up the email/newsletter pair which will set up the template for the actual email.

Update: In order to beat the maximum execution time limit in PHP, I also use

set_time_limit(0);

to remove the limit at the start of my actual script. In addition, it may also be a good idea to run the action as a batch script in the background so that the mass mailing can occur in the background.

It can be tricky sending email from a batch script (e.g. for cron use), here's how to do it. Start with your usual batch setup:

define('SF_ROOT_DIR',    realpath(dirname(__FILE__).'/..'));
define('SF_APP',         'app_name');
define('SF_ENVIRONMENT', 'environment');
define('SF_DEBUG',       true);
 
require_once(SF_ROOT_DIR.DIRECTORY_SEPARATOR.'apps'.DIRECTORY_SEPARATOR.SF_APP.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'config.php');
sfContext::getInstance();

You can set attributes directly from the batch script if you wish:

sfContext::getInstance()->getRequest()->setAttribute('key', $value);

Then forward on to another module/action to handle processing and forwarding to mail as usual:

sfContext::getInstance()->getController()->forward('action', 'module');

To use database connections defined in the databases.yml configuration file, you can use the following snippet:

// initialize database manager
$databaseManager = new sfDatabaseManager();
$databaseManager->initialize();

Create a batch/load_data.php script with:

<?php
 
define('SF_ROOT_DIR',    realpath(dirname(__FILE__).'/..'));
define('SF_APP',         'frontend');
define('SF_ENVIRONMENT', 'dev');
define('SF_DEBUG',       true);
 
require_once(SF_ROOT_DIR.DIRECTORY_SEPARATOR.'apps'.DIRECTORY_SEPARATOR.SF_APP.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'config.php');
 
// initialize database manager
$databaseManager = new sfDatabaseManager();
$databaseManager->initialize();
 
$data = new sfPropelData();
$data->loadData(sfConfig::get('sf_data_dir').DIRECTORY_SEPARATOR.'fixtures');
 
?>

Call it with:

$ cd batch
$ php load_data.php

It will load the data contained in all the .yml files of the data/fixtures/ directory.