//==========================================================================
//
//      if_at91-lwip.c
//
//      lwIP-specific Ethernet driver for Atmel AT91 family on-chip EMAC.
//
//==========================================================================
// ####ECOSGPLCOPYRIGHTBEGIN####                                            
// -------------------------------------------                              
// This file is part of eCos, the Embedded Configurable Operating System.   
// Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc.      
// Copyright (C) 2003, 2004, 2006, 2007 eCosCentric Limited                 
//
// eCos is free software; you can redistribute it and/or modify it under    
// the terms of the GNU General Public License as published by the Free     
// Software Foundation; either version 2 or (at your option) any later      
// version.                                                                 
//
// eCos is distributed in the hope that it will be useful, but WITHOUT      
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or    
// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License    
// for more details.                                                        
//
// You should have received a copy of the GNU General Public License        
// along with eCos; if not, write to the Free Software Foundation, Inc.,    
// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.            
//
// As a special exception, if other files instantiate templates or use      
// macros or inline functions from this file, or you compile this file      
// and link it with other works to produce a work based on this file,       
// this file does not by itself cause the resulting work to be covered by   
// the GNU General Public License. However the source code for this file    
// must still be made available in accordance with section (3) of the GNU   
// General Public License v2.                                               
//
// This exception does not invalidate any other reasons why a work based    
// on this file might be covered by the GNU General Public License.         
// -------------------------------------------                              
// ####ECOSGPLCOPYRIGHTEND####                                              
//==========================================================================
//#####DESCRIPTIONBEGIN####
//
// Author(s):    jlarmour
// Contributors:
// Date:         2007-08-13
// Description:  hardware driver for Atmel AT91 ethernet devices specifically
//               and solely for lwIP TCP/IP stack.
//               Derived directly from if_mcf5272-lwip.c, with some AT91 portions
//               imported from if_at91.c
//
//               At the moment, this driver has only been tested with the SAM7X.
//               It's likely the SAM9 will require further testing due to caching.
//
//####DESCRIPTIONEND####
//==========================================================================

#include <pkgconf/system.h>
#include <pkgconf/hal.h>
#include <pkgconf/devs_eth_arm_at91.h>
#include <pkgconf/io_eth_drivers.h>

#include <cyg/infra/cyg_type.h>
#include <cyg/infra/cyg_ass.h>
#include <cyg/hal/hal_arch.h>
#include <cyg/hal/hal_intr.h>
#include <cyg/hal/hal_cache.h>
#include <cyg/hal/hal_if.h>
#include <cyg/infra/diag.h>
#include <cyg/hal/drv_api.h>
#include <cyg/io/eth/netdev.h>
#include <cyg/io/eth/eth_drv.h>
#include <string.h>
#include <cyg/io/eth_phy.h>

#include <pkgconf/net_lwip.h>
#include "lwip/opt.h"
#include "lwip/err.h"
#include "lwip/netif.h"
#include "lwip/pbuf.h"

#ifdef CYGACC_CALL_IF_DELAY_US
# define AT91_ETH_DELAY_US(us) CYGACC_CALL_IF_DELAY_US(us)
#else
# include <cyg/hal/hal_diag.h>
# define AT91_ETH_DELAY_US(us) HAL_DELAY_US(us)
#endif

// ----------------------------------------------------------------------------
// Debug support, in the form of diagnostics and a simple trace
// buffer. Also make it easy to enable stats without messing about
// with the configuration.

#define DIAG_LEVEL     0
#define DIAG_BUFFERED  0
#define DIAG_WRAP      1
#define DIAG_PHY       0

/* Here's two useful GDB command fragments. The first dumps
 * the rxbds. Change 32 to the RXBD count:
set $foo=0
while ($foo < 32)
p/x at91_eth_info.rxbds[$foo]
set $foo=$foo+1
end

* This one dumps the trace buffer. Change 300 to the
* diag buffer size:
set $foo=at91_eth_trace_next
while ($foo >= 0)
p at91_eth_trace_data[$foo]
set $foo=$foo-1
end
set $foo=300-1
while ($foo > at91_eth_trace_next)
p at91_eth_trace_data[$foo]
set $foo=$foo-1
end
 */


#if (DIAG_LEVEL <= 0)

# define DIAG(_level_, _fmt_, ...)                  \
    CYG_MACRO_START                                 \
    CYG_MACRO_END

# define DIAGPKT(_level_, _msg_, _pkt_, _len_)      \
    CYG_MACRO_START                                 \
    CYG_MACRO_END

#elif (! DIAG_BUFFERED)

#  define DIAG(_level_, _fmt_, ...)                                             \
    CYG_MACRO_START                                                             \
    if ((_level_) <= DIAG_LEVEL) {                                              \
        diag_printf("%s : " _fmt_ "\n", __func__, ## __VA_ARGS__);  \
    }                                                                           \
    CYG_MACRO_END

# define DIAGPKT(_level_, _msg_, _pkt_, _len_)                                  \
    CYG_MACRO_START                                                             \
    if ((_level_) <= DIAG_LEVEL) {                                              \
        diag_printf("%s : %s : pkt %p, len %d\n", __func__, _msg_,     \
                    (cyg_uint8*)_pkt_, _len_);                                  \
    }                                                                           \
    CYG_MACRO_END

#else
// Trace buffer size.
# define DIAG_BUFFER_SIZE      150
typedef struct at91_eth_trace {
    const char*     fn;
    char            msg[100];
    cyg_uint32      len;
    cyg_uint8       packet[14];
} at91_eth_trace_entry;
static at91_eth_trace_entry     at91_eth_trace_data[DIAG_BUFFER_SIZE];
static int                      at91_eth_trace_next      = DIAG_BUFFER_SIZE - 1;
static cyg_bool                 at91_eth_trace_wrapped   = false;

static void
at91_eth_trace(const char* fn, const char* fmt, cyg_uint8* packet, cyg_uint32 len, ...)
{
    at91_eth_trace_entry* entry = &(at91_eth_trace_data[at91_eth_trace_next]);
    va_list ap;

# ifdef DIAG_WRAP
    if (0 == at91_eth_trace_next) {
        at91_eth_trace_next = DIAG_BUFFER_SIZE - 1;
        at91_eth_trace_wrapped = true;
    } else {
        at91_eth_trace_next -= 1;
    }
# else
    if (at91_eth_trace_next < 0) {
        return;
    }
    at91_eth_trace_next -= 1;
# endif

    entry->fn   = fn;
    va_start(ap, len);
    memset(entry->msg, 0, sizeof(entry->msg));
    diag_vsnprintf(entry->msg, sizeof(entry->msg), fmt, ap);
    va_end(ap);
    entry->len  = len;
    if ((cyg_uint8*)0 == packet) {
        memset(entry->packet, 0, 14);
    } else {
        memcpy(entry->packet, packet, 14);
    }
}

#  define DIAG(_level_, _fmt_, ...)                                         \
    CYG_MACRO_START                                                         \
    if ((_level_) <= DIAG_LEVEL) {                                          \
        at91_eth_trace(__func__, _fmt_, (cyg_uint8*)0, 0, ## __VA_ARGS__); \
    }                                                                       \
    CYG_MACRO_END

# define DIAGPKT(_level_, _msg_, _pkt_, _len_)                              \
    CYG_MACRO_START                                                         \
    if ((_level_) <= DIAG_LEVEL) {                                          \
        at91_eth_trace(__func__, _msg_, (cyg_uint8*)_pkt_, _len_);       \
    }                                                                       \
    CYG_MACRO_END

#endif

#if (DIAG_LEVEL < 3)
# define WRITE32(_addr_, _val_)         HAL_WRITE_UINT32(_addr_, _val_)
# define READ32( _addr_, _var_)         HAL_READ_UINT32(_addr_, _var_)
#else

# define WRITE32(_addr_, _val_)                                             \
    CYG_MACRO_START                                                         \
    DIAG(DIAG_LEVEL, "WRITE %s %08x <= 0x%08x", # _addr_, _addr_, _val_) ;  \
    HAL_WRITE_UINT32(_addr_, _val_);                                        \
    CYG_MACRO_END

#define READ32(_addr_, _var_)                                               \
    CYG_MACRO_START                                                         \
    HAL_READ_UINT32(_addr_, _var_);                                         \
    DIAG(DIAG_LEVEL, "READ  %s %08x == 0x%08x", # _addr_, _addr_, _var_) ;  \
    CYG_MACRO_END

#endif

#if 0
// A function that gets placed in RAM. This can be called by flash-resident
// code, e.g. RedBoot, when I need a breakpoint on a specific condition.
static void at91_eth_ramfn(void) __attribute__((section (".2ram.at91_eth_ramfn")));
static int  at91_eth_ramfn_calls;
static void
at91_eth_ramfn(void)
{
    at91_eth_ramfn_calls += 1;
}
#endif

//======================================================================

#if DIAG_PHY
// Include PHY stuff after debug macro defs.
# define phy_debug_printf( __fmt, ... ) DIAG(2, "AT91 ETH: %30s[%4d]: " __fmt, __FUNCTION__, __LINE__, ## __VA_ARGS__ )
#else
# define phy_debug_printf( __fmt, ... )
#endif


#include "at91_phy.h"

// ----------------------------------------------------------------------------
// All AT91s seen so far have a single on-chip ethernet device, so all
// device-specific data could be held in statics. However, in case this
// driver ever gets re-used for a chip with multiple ethernet devices that
// data is accessed via macros which can choose to use statics or access
// the internal data.
//
// The driver currently support a configurable number of rx buffers. The
// size is fixed by the hardware (so far, seen only at 128 bytes). Incoming
// frames are passed into lwIP as pbuf chains.
//
// For tx the current code supports a single outgoing transmission at a
// time. This does limit outgoing bandwidth a bit, but only a bit, and
// saves memory and complexity.

#if defined(CYGPKG_HAL_ARM_ARM9)  // Maybe it works. It's certainly not yet tested.
# error ARM9 support is work in progress
#endif

#if defined(HAL_DCACHE_LINE_SIZE) && (HAL_DCACHE_LINE_SIZE > 4)
# define BD_ALIGNMENT (HAL_DCACHE_LINE_SIZE)
#else
# define BD_ALIGNMENT (4)
#endif

// Buffer descriptors, both rx and tx
typedef struct buffer_descriptor
{
    volatile cyg_uint32 buffer;
    volatile cyg_uint32 sr;
} buffer_descriptor;

#define RXBD_BUFFER_ADDR(rxbd) ((cyg_uint8*)((rxbd).buffer & AT91_EMAC_RBD_ADDR_MASK))

// This is hairy. We need at least 1 more descriptor than SG_LIST indicates.
// But we are also going to be aligning the descriptors according to BD_ALIGNMENT,
// which means that we could waste memory due to the alignment. Instead we would
// rather use that memory for further descriptors. So we round up the descriptor
// count based on how many more we can fit until we reach the alignment
// requirements.
#define UNALIGNED_TX_BUFFER_DESCRIPTOR_SIZE ((CYGNUM_DEVS_ETH_ARM_AT91_TX_BUFS+1)*sizeof(buffer_descriptor))
#define ALIGNED_TX_BUFFER_DESCRIPTOR_SIZE (((UNALIGNED_TX_BUFFER_DESCRIPTOR_SIZE + BD_ALIGNMENT - 1))&~(BD_ALIGNMENT-1))
#define TX_BUFFER_DESCRIPTOR_COUNT ( ALIGNED_TX_BUFFER_DESCRIPTOR_SIZE / sizeof(buffer_descriptor) )

// AT91 Ethernet private data
typedef struct at91_eth
{
    // Interrupt vector details.
    cyg_handle_t    interrupt_handle;
    cyg_interrupt   interrupt_data;

    // Holds pbuf that needs freeing after send.
    struct pbuf *   tx_pbuf;

#ifdef HAL_DCACHE_SIZE
    // The buffer descriptors. Space is declared further below which exists in the
    // cacheable address space. The pointers here though will be set up as pointers into
    // the uncacheable space, rather than computing those every time.
    volatile buffer_descriptor *txbds;
    volatile buffer_descriptor *rxbds;
#endif

    struct netif    netif;
    
    volatile cyg_uint8 tx_index;

    /* Maybe in future we could consider consolidating the booleans below
     * into a single flags bitmask. However, firstly there are potential
     * problems with atomic set/get to be considered, and secondly at present
     * there are only 3 of them - less than one word, so no real gain.
     */
    cyg_uint8          started;
    volatile cyg_uint8 tx_can_send;

    // A flag to indicate whether we ran out of buffers, and need to try and refill on pbuf free.
    cyg_uint8          rx_ran_out_of_buffers;

    // rx_next_buffer tracks the next rx buffer descriptor which the
    // hardware may process.
    cyg_uint8          rx_next_buffer;

    cyg_uint8          mac[6];

    // The storage for the buffer descriptors. Due to calculations for the TX_BUFFER_DESCRIPTOR above,
    // we know the space it occupies is aligned, so no alignment should be required for the
    // rxbds. Defensively, we add alignment anyway - at worst it has no effect.
    // We set the name to have or not have a _c suffix depending on our guess about whether
    // there is data cache support. If there isn't (e.g. ARM7), then we can index the array directly.
    // If there is (e.g. ARM9), then we need to go via an indirect pointer, declared further up.
#ifndef HAL_DCACHE_SIZE
# define txbds_c txbds
# define rxbds_c rxbds
#endif
    volatile buffer_descriptor txbds_c[TX_BUFFER_DESCRIPTOR_COUNT] CYGBLD_ATTRIB_ALIGN(BD_ALIGNMENT);
    volatile buffer_descriptor rxbds_c[CYGNUM_DEVS_ETH_ARM_AT91_RX_BUFS] CYGBLD_ATTRIB_ALIGN(BD_ALIGNMENT);

    // Unlike the txbds, we don't extend the rxbds to fill a whole cache line because this would
    // imply committing more buffer space too. Instead we need padding to guarantee we reach the
    // end of a cache line so the rxbd data is not shared with live data. We cheat using a C99
    // feature to guarantee we don't use any more space than needed (e.g. if it was already
    // aligned at this point).
    char pad[0] CYGBLD_ATTRIB_ALIGN(BD_ALIGNMENT);
} at91_eth;

// The following macros take an at91_eth structure as an argument, for
// a degree of future-proofing should there ever be multiple EMACs.
#define AT91_ETH_INTR_VECTOR(eth)    (CYGNUM_HAL_INTERRUPT_EMAC)
#define AT91_ETH_BASE(eth)           (AT91_EMAC)
#define AT91_ETH_PHY(eth)            (&at91_phy)


// This structure is held in bss so everything is automatically zeroed.
static at91_eth at91_eth_info;

// Define a type for pool used for RX packet data pbufs.
// Unhelpfully lwIP requires the payload to directly follow the pbuf.
// This has consequences for CPUs with caches as we can't share the struct pbuf with the payload
// of any *preceding* packet. This means we waste HAL_DCACHE_LINE_SIZE - sizeof(struct pbuf) bytes :(.
// Fixing this requirement in lwIP would require some major surgery that is best done upstream, and
// has been discussed there so may resolved in due course.
// BD_ALIGNMENT is also suitable to use as the alignment here.
struct at91_pbuf_pool_and_payload {
    struct pbuf pbuf;
    char payload[CYGNUM_LWIP_PBUF_POOL_BUFSIZE];
} CYGBLD_ATTRIB_ALIGN(BD_ALIGNMENT);
struct at91_pbuf_pool_and_payload at91_pbuf_pool[CYGNUM_LWIP_PBUF_POOL_SIZE] CYGBLD_ATTRIB_ALIGN(BD_ALIGNMENT);

#define NEXT_RXBD(current) (((current) == (CYGNUM_DEVS_ETH_ARM_AT91_RX_BUFS - 1)) ? 0 : ((current) + 1))
#define PREV_RXBD(current) (((current) == 0) ? (CYGNUM_DEVS_ETH_ARM_AT91_RX_BUFS - 1) : ((current) - 1))
#define NEXT_TXBD(current) (((current) == (TX_BUFFER_DESCRIPTOR_COUNT - 1)) ? 0 : ((current) + 1))
#define PREV_TXBD(current) (((current) == 0) ? (TX_BUFFER_DESCRIPTOR_COUNT - 1) : ((current) - 1))

/* Various TSR bits may indicate we should clean up a frame
 * which had been sent, and consider it done; whether successfully
 * sent, or if it resulted in an error. These are those bits,
 * since they're used in several places.
 * Used Bit Read (UBR) aka TX Overrun is not one though - encountering
 * this is part of normal operation.
 */
static const cyg_uint32 at91_eth_tsr_tx_done_bits = 
    AT91_EMAC_TSR_COL |
    AT91_EMAC_TSR_RLE |
    AT91_EMAC_TSR_BEX |
    AT91_EMAC_TSR_COMP |
    AT91_EMAC_TSR_UND;

static cyg_uint32 at91_eth_isr(cyg_vector_t vector, cyg_addrword_t data);
static void       at91_eth_dsr(cyg_vector_t vector, cyg_ucount32 count, cyg_addrword_t data);

// ----------------------------------------------------------------------------
// The MAC address. Comes from RedBoot's config info, if CDL indicates we should
// try. Otherwise this will be supplied by the platform HAL, but some
// platforms may not have appropriate hardware. Worst case, we use the CDL-defined
// configurable default value.

static cyg_uint8    at91_eth_default_mac[6]  = { CYGDAT_DEVS_ETH_ARM_AT91_MACADDR } ;

static void
at91_eth_get_config_mac_address(cyg_uint8* mac)
{
    int mac_ok  = 0;
    // Do we have virtual vectors?
#if defined(CYGPKG_DEVS_ETH_ARM_AT91_REDBOOT_HOLDS_ESA) // implies VV support
    // There has been an incompatible change to hal_if.h. Cope with both new and old names
# ifdef CYGNUM_FLASH_CFG_TYPE_CONFIG_ESA    
    mac_ok  = CYGACC_CALL_IF_FLASH_CFG_OP(CYGNUM_CALL_IF_FLASH_CFG_GET, "eth0_esa_data", mac, CYGNUM_FLASH_CFG_TYPE_CONFIG_ESA);
# else    
    mac_ok  = CYGACC_CALL_IF_FLASH_CFG_OP(CYGNUM_CALL_IF_FLASH_CFG_GET, "eth0_esa_data", mac, CYGNUM_FLASH_CFG_OP_CONFIG_ESA);
# endif
#endif
#ifdef CYGHWR_DEVS_ETH_ARM_AT91_GET_ESA
    if (!mac_ok) {
        CYGHWR_DEVS_ETH_ARM_AT91_GET_ESA(mac, mac_ok);
    }
#endif
    if (!mac_ok) {
        memcpy(mac, at91_eth_default_mac, 6);
    }
}

// ----------------------------------------------------------------------------

/* lwIP PBUF_POOL creation override */
__externC struct pbuf *
cyg_at91_eth_lwip_pool_init(void)
{
    struct pbuf *p;
    cyg_ucount32 i;

    for (i=0; i<CYGNUM_LWIP_PBUF_POOL_SIZE; i++)
    {
        p = &at91_pbuf_pool[i].pbuf;
        p->payload = at91_pbuf_pool[i].payload;
        p->next = &at91_pbuf_pool[i+1].pbuf;
        p->len = p->tot_len = CYGNUM_LWIP_PBUF_POOL_BUFSIZE;
        p->flags = PBUF_FLAG_POOL;
    }

    // Last pbuf's next field in fact gets set to NULL to denote end.
    p->next = NULL;

    return &at91_pbuf_pool[0].pbuf;
}

/* Hook when freeing a pool pbuf. */
/* This is called inside a SYS_ARCH_PROTECT, so we should be m-t safe */

__externC void
cyg_at91_eth_lwip_pool_free_hook( struct pbuf *p )
{
    struct  at91_eth*    eth = &at91_eth_info;
    cyg_uint8 i, this_rxbd;
    struct pbuf *new_p;

    CYG_UNUSED_PARAM(struct pbuf *, p);

    DIAG(2, "Freeing pbuf @ 0x%08x, with payload 0x%08x", (unsigned)p, (unsigned)p->payload);

    if (eth->rx_ran_out_of_buffers)
    {
        // Look through the complete list of rxbds for buffers that need refilling. Start
        // from the next rx buffer, as we need to prioritise rxbds that will be reused
        // sooner.
        for (i=CYGNUM_DEVS_ETH_ARM_AT91_RX_BUFS, this_rxbd=eth->rx_next_buffer; i > 0; i--)
        {
            // In the deliver function, we set the buffer field to
            // NULL if the previous pool alloc failed, so that's what to look for.
            if (RXBD_BUFFER_ADDR(eth->rxbds[this_rxbd]) == NULL) {
                new_p = pbuf_alloc(PBUF_RAW, CYGNUM_LWIP_PBUF_POOL_BUFSIZE, PBUF_POOL);
                if (!new_p) {
                    DIAG(2, "No pool pbufs (yet) to refill rxbd[%d]", this_rxbd);
                    // Nothing more we can do with this one or any others.
                    // rx_ran_out_of_buffers will remain set.
                    break;
                }
                DIAG(2, "Refilled rxbd[%d] with pbuf @ 0x%08x (payload 0x%08x)", this_rxbd,
                     (unsigned)new_p, new_p ? (unsigned)(new_p->payload) : 0);
                eth->rxbds[this_rxbd].sr = 0;
                // Preserve wrap field, assign physical address of buffer to use, and owner can now be emac
                eth->rxbds[this_rxbd].buffer = (eth->rxbds[this_rxbd].buffer & AT91_EMAC_RBD_ADDR_WRAP) |
                    ((cyg_uint32)CYGARC_PHYSICAL_ADDRESS(new_p->payload)) | 
                    AT91_EMAC_RBD_ADDR_OWNER_EMAC;
            }
            this_rxbd = NEXT_RXBD(this_rxbd);
        }

        if (0 == i) {
            // so we did scan through all the rxbds and successfully allocate any empty buffers. Good!
            eth->rx_ran_out_of_buffers = 0;
        }
    }
}

//======================================================================
// Transmitter/Receiver enable

static void 
at91_enable(at91_eth *eth)
{
   cyg_uint32 ctl;

   READ32(AT91_ETH_BASE(eth) + AT91_EMAC_NCR, ctl);
   ctl |= AT91_EMAC_NCR_RE | AT91_EMAC_NCR_TX;
   WRITE32(AT91_ETH_BASE(eth) + AT91_EMAC_NCR, ctl);
}

static void
at91_start_transmitter(at91_eth *eth)
{
   cyg_uint32 ctl;

   /* For the future: enable CLKEN in USRIO here, and disable when transmitter goes idle */

   READ32(AT91_ETH_BASE(eth) + AT91_EMAC_NCR, ctl);
   ctl |= AT91_EMAC_NCR_TSTART;
   WRITE32(AT91_ETH_BASE(eth) + AT91_EMAC_NCR, ctl);
}

// ----------------------------------------------------------------------------

// Configure the pins so that the EMAC has control of them. 
static void
at91_cfg_pins(void)
{
   // FIXME: this should come from the platform. But this is hard
   // to co-ordinate with a moving anoncvs.
   // TODO: Add option and code to select RMII vs MII
   
#if defined(CYGHWR_HAL_ARM_AT91SAM7)
   // This assumes the MII is used, not the RMII    
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_EREFCK);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ECRS);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ECOL);

   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ERXDV);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ERX0);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ERX1);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ERX2);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ERX3);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ERXER);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ERXCK);

   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ETXEN);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ETX0);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ETX1);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ETX2);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ETX3);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_ETXER);

   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_EMDC);
   HAL_ARM_AT91_PIO_CFG(AT91_EMAC_EMDIO);

   HAL_ARM_AT91_GPIO_CFG_DIRECTION(AT91_GPIO_PB18, AT91_PIN_OUT);
   HAL_ARM_AT91_GPIO_RESET(AT91_GPIO_PB18);
   
#elif defined(CYGHWR_HAL_ARM_ARM9_SAM9_SAM9260)

   // This assumes RMII is used
   WRITE32( SAM9_PIOA+_PIO_PDR, 0x003FF000 );
   WRITE32( SAM9_PIOA+_PIO_ASR, 0x003FF000 );

#else

#error Unknown AT91 variant
   
#endif
}

//======================================================================
// Set a specific address match to a given address. Packets received which
// match this address will be passed on.

static void
at91_set_mac(at91_eth *eth, const cyg_uint8 *enaddr, int sa)
{
   cyg_uint32 hi, lo;

   CYG_ASSERTC(sa > 0 && sa < 5);
   sa--;

   lo = ((enaddr[3] << 24) |
         (enaddr[2] << 16) |
         (enaddr[1] <<  8) |
         (enaddr[0]));

   hi = ((enaddr[5] <<  8) |
         (enaddr[4]));

   WRITE32(AT91_ETH_BASE(eth) + AT91_EMAC_SA1L + (8*sa), lo);
   WRITE32(AT91_ETH_BASE(eth) + AT91_EMAC_SA1H + (8*sa), hi);
}

//======================================================================

static void
at91_clear_stats(at91_eth *eth)
{
   cyg_uint32 ctl;

   READ32(AT91_ETH_BASE(eth) + AT91_EMAC_NCR, ctl);
   ctl |= AT91_EMAC_NCR_CSR;
   WRITE32(AT91_ETH_BASE(eth) + AT91_EMAC_NCR, ctl);
}

//======================================================================

__externC void cyg_lwip_eth_ecos_init(void)
{
    struct at91_eth*        eth = &at91_eth_info;
    cyg_uint32              i;
    struct pbuf *           p;
    err_t err;
    cyg_uint32 usrio, ncfg;
    unsigned short phy_state = 0;
    char do_dhcp;
    const unsigned char enzero[6] = { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};

    DIAG(1, "entry");
    
    // First, some basic hardware initialisation (possibly re-initialisation)
    // of the clocks, so the subsystem is powered up.
#if defined(CYGHWR_HAL_ARM_AT91SAM7)
    // Enable the clock to the EMAC
    WRITE32(AT91_PMC + AT91_PMC_PCER, AT91_PMC_PCER_EMAC);
    // Enable clocks to PIO banks (probably unnecessary)
    WRITE32(AT91_PMC + AT91_PMC_PCER, AT91_PMC_PCER_PIOB);
    WRITE32(AT91_PMC + AT91_PMC_PCER, AT91_PMC_PCER_PIOA);
#elif defined(CYGHWR_HAL_ARM_ARM9_SAM9_SAM9260)
    WRITE32(_PMC_PCER, (1<<21));
#endif

    // Set pin functions correctly
    at91_cfg_pins();

#ifdef CYGSEM_HAL_USE_ROM_MONITOR
    // If we are debugging over RedBoot, it is possible that there is an
    // outgoing packet still queued up. Give it a chance to get out, a
    // couple of milliseconds should suffice. The preceding initialisation
    // should not conflict with this.
    for (i = 0; i < 20; i++) {
        cyg_uint32 tsr;
        READ32( AT91_ETH_BASE(eth) + AT91_EMAC_TSR, tsr );
        if (0 == (tsr & AT91_EMAC_TSR_TGO )) {
            break;
        }
        HAL_DELAY_US(100);
    }
#endif

    /* Enable  IO Clock */
    usrio = AT91_EMAC_USRIO_CLKEN;
#if !defined(CYGHWR_HAL_ARM_AT91SAM7)
    usrio |= AT91_EMAC_USRIO_RMII;
#endif
    WRITE32(AT91_ETH_BASE(eth)+AT91_EMAC_USRIO, usrio);

    // Reset main registers to default sane state with all operations terminated
    // before we continue, in case something else had been playing with them
    // before, e.g. RedBoot. Status bits will be cleared later after the below has
    // had a chance to take effect, in case there _was_ an effect.
    WRITE32( AT91_ETH_BASE(eth) + AT91_EMAC_IDR, 0x00003FFF ); // Mask all interrupts
    WRITE32( AT91_ETH_BASE(eth) + AT91_EMAC_NCR, 0 ); 
    WRITE32( AT91_ETH_BASE(eth) + AT91_EMAC_NCFG, 0x800 );

    // All interrupts were masked further up, so it's safe to attach interrupt handler.
    cyg_drv_interrupt_create(AT91_ETH_INTR_VECTOR(eth),
                             CYGNUM_DEVS_ETH_ARM_AT91_INTRPRI,
                             (CYG_ADDRWORD) eth,
                             at91_eth_isr,
                             at91_eth_dsr,
                             &eth->interrupt_handle,
                             &eth->interrupt_data);

    cyg_drv_interrupt_attach(eth->interrupt_handle);
    // Unmask overall eth interrupt (even though all individual sources
    // remain masked within EMAC)
    cyg_drv_interrupt_unmask(AT91_ETH_INTR_VECTOR(eth));

    // Determine the MAC address. This is done by either RedBoot or the platform HAL or CDL.
    at91_eth_get_config_mac_address(eth->mac);

    DIAG(1, "Using ESA %02x:%02x:%02x:%02x:%02x:%02x",
         eth->mac[0],eth->mac[1],eth->mac[2],
         eth->mac[3],eth->mac[4],eth->mac[5]);

    // Give the EMAC its address
    at91_set_mac(eth, eth->mac, 1);
    at91_set_mac(eth, enzero, 2);
    at91_set_mac(eth, enzero, 3);
    at91_set_mac(eth, enzero, 4);

    // Now sort out the buffers and buffer descriptors for both TX and RX.

#ifdef HAL_DCACHE_SIZE
    // Store the uncached buffer descriptor addresses to save recomputing them repeatedly.
    eth->txbds = (volatile buffer_descriptor *)CYGARC_UNCACHED_ADDRESS(&eth->txbds_c[0]);
    eth->rxbds = (volatile buffer_descriptor *)CYGARC_UNCACHED_ADDRESS(&eth->rxbds_c[0]);
#endif

    // We can fill in the buffer fields in the various rx buffer descriptors.
    for (i = 0; i < CYGNUM_DEVS_ETH_ARM_AT91_RX_BUFS; i++) {
        p = pbuf_alloc(PBUF_RAW, CYGNUM_LWIP_PBUF_POOL_BUFSIZE, PBUF_POOL);
        if (!p) {
            // By rights this could be an assert, due to the CDL requirement on PBUF_POOL_SIZE.
            // But for now I'll make it a soft error, and simply return, failing to start
            // the device.
            DIAG(1, "Failed to allocate initial rx pool pbufs");
            return;
        }
        // We don't need to track p itself - because we have complete control of the
        // pool we can derive it later.
        eth->rxbds[i].buffer = ((cyg_uint32)CYGARC_PHYSICAL_ADDRESS(p->payload)) | AT91_EMAC_RBD_ADDR_OWNER_EMAC;
        eth->rxbds[i].sr     = 0;
    }
    // Set wrap bit in last rxbd.
    eth->rxbds[i-1].buffer |= AT91_EMAC_RBD_ADDR_WRAP;
    // And tell the EMAC where the first receive buffer descriptor is
    WRITE32( AT91_ETH_BASE(eth) + AT91_EMAC_RBQP, CYGARC_PHYSICAL_ADDRESS(&eth->rxbds_c[0]));

    // Ditto for the tx buffers.
    for (i=0; i < TX_BUFFER_DESCRIPTOR_COUNT; i++)
    {
        eth->txbds[i].buffer = 0; // nothing to put there yet
        eth->txbds[i].sr     = AT91_EMAC_TBD_SR_USED;
    }
    // Set wrap bit in last txbd.
    eth->txbds[i-1].sr |= AT91_EMAC_TBD_SR_WRAP;
    // And tell the EMAC where the first transmit buffer descriptor is
    WRITE32( AT91_ETH_BASE(eth) + AT91_EMAC_TBQP, CYGARC_PHYSICAL_ADDRESS(&eth->txbds_c[0]));

    eth->tx_can_send = 1;

    // Set up the PHY
    CYG_ASSERTC(AT91_ETH_PHY(eth));

    at91_mdio_enable();
   
    if (!_eth_phy_init(AT91_ETH_PHY(eth)))
    {
        at91_mdio_disable();
        return;
    }

    // On powerup, on the SAM9260EK, the PHY seems to have difficulty
    // coming up and negotiating with the other end. The following is
    // an attempt to fix this by resetting the PHY and then restarting
    // autonegotiation.  Unfortunately none of this seems to be
    // reliable, but the PHY can eventually be brought up by
    // power-cycling the board; the die appears to be cast before any
    // software gets to run.
    {
        int i;
#if 0 // This doesn't seem needed any more on the SAM7X-EK, so perhaps nor on SAM9260EK?
        unsigned short reg;

        // Reset the PHY
        _eth_phy_write(AT91_ETH_PHY(eth), 0, AT91_ETH_PHY(eth)->phy_addr, (1<<15));

        DIAG(2, "AT91_ETH: Waiting for PHY to reset");
        for( i = 0; i < 20; i++ )
        {
            AT91_ETH_DELAY_US(100000);   // 1/10 second
            DIAG(2, " ....still waiting for reset....");
            _eth_phy_read(AT91_ETH_PHY(eth), 0, AT91_ETH_PHY(eth)->phy_addr, &reg );
            if ((reg & (1<<15)) == 0)
                break;
        }

        // Force an autonegotiation
        _eth_phy_read(AT91_ETH_PHY(eth), 0, AT91_ETH_PHY(eth)->phy_addr, &reg );
        reg |= (1<<12)|(1<<9);   // Set autonegotiation bits
        _eth_phy_write(AT91_ETH_PHY(eth), 0, AT91_ETH_PHY(eth)->phy_addr, reg );
#endif       
        DIAG(2, "AT91_ETH: Waiting for link to come up");
        for( i = 0; i < 20; i++ )
        {
            AT91_ETH_DELAY_US(100000);   // 1/10 second
            DIAG(2, "....still waiting for link....");
            phy_state = _eth_phy_state(AT91_ETH_PHY(eth));
            if ((phy_state & ETH_PHY_STAT_LINK) != 0)
                break;
        }
    }

#if 0   
    // Get the current mode from PHY, set it in EMAC and print it
    phy_state = _eth_phy_state(AT91_ETH_PHY(eth));
#endif

    at91_mdio_disable();

    DIAG(2, "phy_state %04x", phy_state);

    READ32(AT91_ETH_BASE(eth) + AT91_EMAC_NCFG, ncfg);

    if ((phy_state & ETH_PHY_STAT_LINK) != 0)
    {
        if (((phy_state & ETH_PHY_STAT_100MB) != 0))
        {
            DIAG(1, "AT91_ETH: 100Mbyte/s");
            ncfg |= AT91_EMAC_NCFG_SPD_100Mbps;
        }
        else
        {
            DIAG(1,"AT91_ETH: 10Mbyte/s");
            ncfg &= ~(AT91_EMAC_NCFG_SPD_100Mbps);
        }
        if((phy_state & ETH_PHY_STAT_FDX))
        {
            DIAG(1,"AT91_ETH: Full Duplex");
            ncfg |= AT91_EMAC_NCFG_FD;
        }
        else
        {
            DIAG(1,"AT91_ETH: Half Duplex");
            ncfg &= ~(AT91_EMAC_NCFG_FD);
        }
    }
    else
    {
        DIAG(1,"AT91_ETH: No Link");
    }

    //Setup the network configuration
    ncfg |= (AT91_EMAC_NCFG_RLCE|AT91_EMAC_NCFG_DRFCS);

    WRITE32(AT91_ETH_BASE(eth) + AT91_EMAC_NCFG, ncfg);

    // Clear the Statistics counters;
    at91_clear_stats(eth);

    // Clear all status bits so nothing old remains.
    WRITE32( AT91_ETH_BASE(eth) + AT91_EMAC_TSR, 0xffffffff ); // clear all status bits
    WRITE32( AT91_ETH_BASE(eth) + AT91_EMAC_RSR, 0xffffffff ); // clear all status bits
    READ32( AT91_ETH_BASE(eth) + AT91_EMAC_ISR, ncfg); // clear int status by reading

    // Enable interrupts we are interested in.
    // Interrupts for successful operations, and a variety of error conditions.
    WRITE32( AT91_ETH_BASE(eth) + AT91_EMAC_IER, 
             AT91_EMAC_ISR_RCOM | AT91_EMAC_ISR_TCOM |
             AT91_EMAC_ISR_TUND | AT91_EMAC_ISR_RTRY |
             AT91_EMAC_ISR_TBRE | AT91_EMAC_ISR_ROVR |
             AT91_EMAC_ISR_HRESP
             );

    DIAG(1, "calling higher-level");
    
    // Initialize the upper level driver
    
    err = cyg_lwip_eth_drv_init_netif((void *)eth, eth->mac, &eth->netif, &do_dhcp);

    if (ERR_OK == err)
    {
        // Enable rx/tx operation.
        at91_enable(eth);

        eth->started = 1;

#ifdef CYGFUN_LWIP_DHCP
        //
        // we call this after the driver was started successfully
        //
        if (do_dhcp)
            cyg_lwip_dhcp_init(&eth->netif);
#endif
    }

    DIAG(1, "done");
}

// ----------------------------------------------------------------------------
// The RX interrupt triggers whenever a whole frame has been received (or
// when an error has occurred). The hardware may be busy receiving into the
// next buffer.
//
// Because of the ring buffer several packets may arrive before the cpu gets
// around to processing them. It is possible to save a little bit of load
// by masking the interrupt inside the ethernet device, then doing the
// unmask inside the deliver() code.
//
// The TX interrupt triggers when a whole frame has been sent. Since at present
// we only send one frame at a time, the TX hardware will be idle.

static cyg_uint32
at91_eth_isr(cyg_vector_t vector, cyg_addrword_t data)
{
    at91_eth *eth = (at91_eth *)data;

    CYG_UNUSED_PARAM(at91_eth *, eth); // potentially unused due to macro use below

    cyg_drv_interrupt_mask(AT91_ETH_INTR_VECTOR(eth));

    // Ack now, to prevent issues with other interrupts.
    cyg_drv_interrupt_acknowledge(AT91_ETH_INTR_VECTOR(eth));

    return (CYG_ISR_HANDLED|CYG_ISR_CALL_DSR);  // Run the DSR
}

static void
at91_eth_dsr(cyg_vector_t vector, cyg_ucount32 count, cyg_addrword_t data)
{
    // There is no need to worry about cleaning up the buffer descriptors.
    // The hardware will have set the ownership bit, which is the only one
    // which needs handling.
    DIAG(1, "AT91 ETH DSR");

    cyg_lwip_eth_dsr();
}

// ----------------------------------------------------------------------------

// The driver only supports one transmit at a time, and a global is used
// to keep track of whether or not a transmit is in progress.
static __inline__ int
at91_eth_can_send(struct at91_eth* eth)
{
    return eth->tx_can_send;
}

// Perform deliveries on the transmit side. Called in delivery thread context.
// check_tsr is set if we should look at the tsr to see if anything needs doing;
// otherwise we consider the last packet finished regardless.

static void
at91_eth_deliver_tx(at91_eth *eth, cyg_bool check_tsr)
{
    cyg_uint8 index;

    if (check_tsr) {
        cyg_uint32 tsr, masked_tsr;

        // Check if any bits are set in TSR which indicate we should
        // consider the outstanding tx frame as done (whether by success/error).

        READ32(AT91_ETH_BASE(eth)+AT91_EMAC_TSR, tsr);
        masked_tsr = tsr & at91_eth_tsr_tx_done_bits;
        if (0 == masked_tsr)
            return;
        // Clear those bits in the TSR
        WRITE32(AT91_ETH_BASE(eth)+AT91_EMAC_TSR, masked_tsr);
    }

    DIAG(1, "tx done");

    // Only need to free if the original pbuf was used and therefore pbuf_ref()d.
    if (eth->tx_pbuf)
    {
        pbuf_free(eth->tx_pbuf);
        eth->tx_pbuf = NULL;
    }
    else
    {
#ifdef CYGPKG_INFRA_DEBUG
        // This is an important thing to catch, but could arise for all sorts
        // of reasons.
        diag_printf("AT91 eth: delivered tx but no tx_pbuf?! Tell jifl\n");
#endif
    }

    // Ensure all txbds that constituted the frame just sent get used bit set.
    // Otherwise the next frame we send may be followed by a txbd with the
    // used bit clear.
    // We do this by working backwards from the current tx index until we
    // find something with the used bit set - this will be the first fragment
    // of the frame.
    for (index = PREV_TXBD(eth->tx_index); 
         0 == (eth->txbds[index].sr & AT91_EMAC_TBD_SR_USED); // found the start?
         index = PREV_TXBD(index))
    {
        eth->txbds[index].sr |= AT91_EMAC_TBD_SR_USED; // ORed to protect wrap bit.
    }

    eth->tx_can_send = 1;
}

//
// cyg_lwip_eth_low_level_output():
//
// This is the function called by higher layers which should do the actual
// transmission of the packet. The packet is contained in the pbuf that is
// passed to the function. This pbuf might be chained.


__externC err_t
cyg_lwip_eth_low_level_output(struct netif *netif, struct pbuf *p)
{
    at91_eth *eth = (at91_eth *)netif->state;
    cyg_uint32 index;
    cyg_uint32 last_index;
    cyg_uint32 pbuf_chain_length=1;
    struct pbuf *q;

    // Sanity check p. Don't waste code returning on an error like this.
    // Just don't screw up in the first place!
    if (!p)
    {
        DIAG(1, "cyg_lwip_eth_low_level_output passed NULL pbuf");
        CYG_FAIL("cyg_lwip_eth_low_level_output passed NULL pbuf");
    }

    if (!p->len || !p->tot_len)
    {
        DIAG(1, "cyg_lwip_eth_low_level_output passed empty pbuf");
        CYG_FAIL("cyg_lwip_eth_low_level_output passed empty pbuf");
    }

    if (!eth->started)
        return ERR_IF;

    // Can we send?
    if (at91_eth_can_send(eth) <= 0)
    {
        DIAG(2, "could not send");

        // If we can't send, we need to call the delivery function to make
        // sure that the reason we can't send isn't just because we are
        // either higher priority than the normal thread doing delivery,
        // and thus preventing it delivering; or that we _are_ the
        // delivery thread, in which case we would never see it complete
        // even if it had, because clearly this thread has been "doing
        // stuff" if it has stuff to send.
        at91_eth_deliver_tx(eth, true);

#ifdef CYGIMP_IO_ETH_DRIVERS_LWIP_TX_FULL_WAIT
        while (at91_eth_can_send(eth) <= 0)
        {
            // As above, call the delivery function each time. We may
            // be the delivery thread anyway, so if we didn't, looping here
            // and not calling it would otherwise cause everything to freeze.
            at91_eth_deliver_tx(eth, true);

# if (CYGNUM_IO_ETH_DRIVERS_LWIP_TX_FULL_WAIT_DELAY > 0)
            // Give others a chance to run
            AT91_ETH_DELAY_US(CYGNUM_IO_ETH_DRIVERS_LWIP_TX_FULL_WAIT_DELAY);
# endif
        }
#else // !CYGIMP_IO_ETH_DRIVERS_LWIP_TX_FULL_WAIT
        /* Otherwise if we still can't send, just drop the packet completely. 
         * We claim success because there isn't necessarily really an error as such.
         * We may just be generating data too fast.
         */
        if (at91_eth_can_send(eth) <= 0) {
            DIAG(1, "Couldn't send packet. Dropping");
            return ERR_OK;
        }
#endif
        DIAG(1, "Can now send");
    }

    // Quickly need to check the pbuf chain is not too long for us.
    // We could probably do this at the same time as the loop where
    // we fill in the txbds below, but to be honest, the chains should
    // usually be short enough (typically not a chain at all!) that 
    // this is faster.
    for (q=p->next; q; q=q->next)
        if (q->len)
            pbuf_chain_length++;

    if (pbuf_chain_length >= (TX_BUFFER_DESCRIPTOR_COUNT-1))
        return ERR_IF;

    eth->tx_can_send    = 0;
    /* In the following line, last_index doesn't really need set, but it
     * silences a compiler warning.
     */
    last_index = index = eth->tx_index;

    // incr ref count of pbuf, so that caller's free will not really free up
    // memory. At least not until packet is definitely sent.
    pbuf_ref(p);
    eth->tx_pbuf = p;

    for (q=p; q; q=q->next)
    {
        // Don't waste a txbd on empty chain entries
        if (q->len) {
            eth->txbds[index].buffer = CYGARC_PHYSICAL_ADDRESS(q->payload);
            // We can leave USED bit clear always because a new send is only
            // initiated when the transmit engine is idle.
            eth->txbds[index].sr = (eth->txbds[index].sr & AT91_EMAC_TBD_SR_WRAP) |    // preserve wrap flag if set
                (cyg_uint32)(q->len & AT91_EMAC_TBD_SR_LEN_MASK);
            last_index = index;
            index = NEXT_TXBD(index);
        }
    }

    // Set End Of Frame bit in the last descriptor
    eth->txbds[last_index].sr |= AT91_EMAC_TBD_SR_EOF;

    eth->tx_index = index;

    // Ensure writeback cache data is written to physical memory
#ifdef HAL_DCACHE_SYNC
    HAL_DCACHE_SYNC();
#endif
                
    DIAGPKT(1, "start", p->payload, p->tot_len);
    
    // Set transmitter going
    at91_start_transmitter(eth);

    return ERR_OK;
}

// ----------------------------------------------------------------------------
// Perform deliveries on the RX side. Called in delivery thread context.

static void
at91_eth_deliver_rx( at91_eth *eth )
{
    int cache_state;
    int current_rxbd;
    int start_of_frame = -1;
    cyg_uint32 sr;
    cyg_uint32 owner;


    DIAG(1, "rx_rdy");

    // Invalidate the cache's view of the rx buffer data space as this will
    // have changed. The rx buffer descriptors will be ok as they are accessed
    // via uncached space, and are in their own cache lines; but we have to
    // invalidate everything anyway. Since we invalidate, we need to sync dirty
    // cache lines to memory first.
    //
    // This could be done more selectively, but apparently the HAL_DCACHE_STORE
    // and HAL_DCACHE_INVALIDATE implementations don't work for ARM9 (and so
    // aren't defined). And for ARM7 these are no-ops.

    // If cache is always off (ARM7), cache_state will be set to constant 0 and
    // compiler should optimise the rest away hopefully, so we don't even get
    // the interrupt disable blip.
    HAL_DCACHE_IS_ENABLED(cache_state);
        
    if (cache_state) {
        cyg_drv_isr_lock(); // Need ISRs off during cache sync/invalidate
        HAL_DCACHE_SYNC();
        HAL_DCACHE_INVALIDATE_ALL();
        cyg_drv_isr_unlock();
    }

    // Loop while there are non-empty packets. This has to be done with a bit
    // of care. Packets should be processed in the right order where possible,
    // which means keeping track of the last packet received.
    current_rxbd = eth->rx_next_buffer;
    for (;;) {
        
        // We work on the assumption that the driver and the hardware agree
        // which rxbd is being filled next. Things will go wrong if this isn't true.
        // So we always assume the current rxbd should be the start, and you keep
        // going till you reached the End-Of-Frame.
        //
        // The exception is for error frames, where there can be starts of frames
        // but no end. This would be indicated by reaching either a new start of
        // frame or an empty rxbd. (An empty rxbd means it is pending rx and thus
        // marked as owned by EMAC, not software).
        // We also count rxbds with NULL buffer as empty, as those are buffers where
        // we weren't able to reallocate a pbuf to replace the payload. The EMAC
        // will halt (and would report BNA if we let it) if one is reached.
        
        owner = eth->rxbds[current_rxbd].buffer & AT91_EMAC_RBD_ADDR_OWNER_SW;
        sr = eth->rxbds[current_rxbd].sr;

        if ( owner == AT91_EMAC_RBD_ADDR_OWNER_EMAC )
        {
            if ( start_of_frame >= 0 )
            {
                // We had found a start of frame. There are two possible reasons
                // we could find an empty rxbd now:
                // 1) The frame is in transit, and the EMAC is still filling the buffers.
                // 2) It's an error frame and will never complete.
                // We would purge it if we could be sure about 2, but it might be 1.
                // So we leave it for now. The next RX interrupt on frame completion
                // will cause us to notice.

                DIAG(2, "Had found SOF @ rxbd[%d], but now found empty rxbd[%d] - in transit?",
                     start_of_frame, current_rxbd);
                // leave eth->rx_next_buffer untouched
            }
            return;
        } // if

        // Otherwise, this is owned by software

        // If payload is NULL, then the hardware will also have stopped here, and (probably)
        // have indicated BNA, if a new frame arrived. We have a go allocating a pbuf again,
        // just in case.
        // Any partially received frame is toast for definite.
        if ( NULL == RXBD_BUFFER_ADDR(eth->rxbds[current_rxbd]) )
        {
            struct pbuf *newp;

            // Return any packet fragments that had started to be part of a frame to the hardware.
            if (start_of_frame >= 0)
            {
                for ( ; start_of_frame != current_rxbd; start_of_frame = NEXT_RXBD(start_of_frame) )
                {
                    eth->rxbds[start_of_frame].sr = 0;
                    eth->rxbds[start_of_frame].buffer &= ~AT91_EMAC_RBD_ADDR_OWNER_SW;
                }
            }

            newp = pbuf_alloc(PBUF_RAW, CYGNUM_LWIP_PBUF_POOL_BUFSIZE, PBUF_POOL);

            DIAG(2, "After hitting sw-owned NULL packet at rxbd[%d], attempting refill with pbuf @ 0x%08x (payload 0x%08x)",
                 current_rxbd, (unsigned)newp, newp ? (unsigned)(newp->payload) : 0);

            if (newp) {
                // we can only really replace it if we succeeded of course.
                eth->rxbds[current_rxbd].sr = 0;
                eth->rxbds[current_rxbd].buffer = (cyg_uint32)newp->payload | (eth->rxbds[current_rxbd].buffer & AT91_EMAC_RBD_ADDR_WRAP);
            }

            eth->rx_next_buffer = current_rxbd; // hardware will start from here next time
            return; // and stop, since the hardware will also have.
        }

        if ( sr & AT91_EMAC_RBD_SR_SOF )
        {
            // start of a new frame
            DIAG(1, "Found SOF at rxbd[%d] (old SOF==rxbd[%d])", current_rxbd, start_of_frame);
            if ( start_of_frame >= 0 )
            {
                // Erk. We already found a start of frame. That must mean the old one
                // is bogus. Probably a bad packet.
                // We need to kill those packet fragments. This is easy - just set the
                // rxbd as owned by the EMAC again.

                for ( ; start_of_frame != current_rxbd; start_of_frame = NEXT_RXBD(start_of_frame) )
                {
                    eth->rxbds[start_of_frame].sr = 0;
                    eth->rxbds[start_of_frame].buffer &= ~AT91_EMAC_RBD_ADDR_OWNER_SW;
                }
            }
            else
                start_of_frame = current_rxbd;
            eth->rx_next_buffer = start_of_frame; // new baseline
        } // if


        // End of frame. A whole valid packet.
        if ( sr & AT91_EMAC_RBD_SR_EOF )
        {
            struct pbuf *p, *first_pbuf, *newp;
            int len = sr & AT91_EMAC_RBD_SR_LEN_MASK;

            // Found end of frame. Might be same slot as SOF, and that's fine.
            // But if we didn't find the start....
            if ( start_of_frame < 0 )
            {
                // Oh dear. We found EOF, but no SOF. Something's gone wrong.
                DIAG(1, "Eek! EOF at rxbd[%d] but no SOF!", current_rxbd);
                CYG_FAIL("RXBD EOF but no SOF");
            }

            DIAG(2, "Detected whole frame. SOF @ rxbd[%d], EOF @ rxbd[%d], len %d",
                 start_of_frame, current_rxbd, len);

            // rewind rxbd to start of frame. We don't mind since we know we've got a consistent frame.
            current_rxbd = start_of_frame;

            // The pbuf structure always precedes the pbuf payload, from the way we set the pool up.
            p = first_pbuf = (struct pbuf *)(RXBD_BUFFER_ADDR(eth->rxbds[start_of_frame]) - sizeof(struct pbuf));

            first_pbuf->tot_len = len;

            for (;;) {
                if ( p->payload != RXBD_BUFFER_ADDR(eth->rxbds[current_rxbd]) )
                {
                    DIAG(1, "Eeek! Expected pbuf (==%08x) payload (==%08x) is not what's in rxbd[%d] (==%08x)",
                         (unsigned)p, (unsigned)p->payload, current_rxbd, eth->rxbds[current_rxbd].buffer);
                    CYG_FAIL("Pbuf payload mismatch with rxbd buffer");
                }

                sr = eth->rxbds[current_rxbd].sr;

                DIAG(1, "Adding pbuf 0x%08x from rxbd[%d] with buffer 0x%08x length %d sr 0x%08x to chain",
                     (unsigned)p, current_rxbd, (unsigned)RXBD_BUFFER_ADDR(eth->rxbds[current_rxbd]),
                     (sr & AT91_EMAC_RBD_SR_LEN_MASK) ? (int)sr & AT91_EMAC_RBD_SR_LEN_MASK : CYGNUM_LWIP_PBUF_POOL_BUFSIZE,
                     sr);

                // Put back a replacement pool pbuf payload into the rxbd
                newp = pbuf_alloc(PBUF_RAW, CYGNUM_LWIP_PBUF_POOL_BUFSIZE, PBUF_POOL);

                if (newp) {
                    // we can only really replace it if we succeeded of course.
                    eth->rxbds[current_rxbd].buffer = (cyg_uint32)newp->payload | (eth->rxbds[current_rxbd].buffer & AT91_EMAC_RBD_ADDR_WRAP);
                } else {
                    DIAG(1, "No pool pbufs left, keeping rxbd[%d] as non-empty", current_rxbd);
                    // Set buffer as NULL and don't mark empty, so we know later which rxbds need refilling.
                    // We preserve wrap and SW bits.
                    eth->rxbds[current_rxbd].buffer &= AT91_EMAC_RBD_ADDR_WRAP|AT91_EMAC_RBD_ADDR_OWNER_SW;
                    eth->rx_ran_out_of_buffers = 1;
                }

                if ( 0 != (sr & AT91_EMAC_RBD_SR_EOF) ) {
                    // since it's the last, let's check the length
                    if ( len > CYGNUM_LWIP_PBUF_POOL_BUFSIZE )
                    {
                        DIAG(1, "Eeek! Length %d unexpectedly too large", len);
                        CYG_FAIL("When reached EOF fragment, remaining length was too large");
                    }
                    p->len = len;
                    p->next = NULL;
                    break; // finished the frame
                } else {
                    if ( 0 != (sr & AT91_EMAC_RBD_SR_LEN_MASK) )
                    {
                        DIAG(1, "Eeeek! Length from sr %08x not 0", sr);
                        CYG_FAIL("Frag length not 0");
                    }
                    p->len = CYGNUM_LWIP_PBUF_POOL_BUFSIZE;
                    len -= CYGNUM_LWIP_PBUF_POOL_BUFSIZE;

                    // Now link to payload of next rxbd in frame
                    current_rxbd = NEXT_RXBD(current_rxbd);
                    // The pbuf structure always precedes the pbuf payload, from the way we set the pool up.
                    p->next = (struct pbuf *)(RXBD_BUFFER_ADDR(eth->rxbds[current_rxbd]) - sizeof(struct pbuf));
                    p = p->next;
                    p->tot_len = len;
                } // else
            } // for
            DIAGPKT(1, "packet received", first_pbuf->payload, first_pbuf->tot_len);

            // Send this packet, i.e. pbuf chain, up.
            cyg_lwip_eth_drv_ecosif_input( &eth->netif, first_pbuf );

            // Reset pointers for next packet
            current_rxbd = eth->rx_next_buffer = NEXT_RXBD(current_rxbd);
            start_of_frame = -1;
        } // if
        else // Not end of frame - a starting or intermediate packet frag
        {
            if (start_of_frame < 0) {
                DIAG(1, "Eek! found packet frag at rxbd[%d] with no start found", current_rxbd);
                CYG_FAIL("Packet frag found with no start");
            }
            current_rxbd = NEXT_RXBD(current_rxbd);
        }
    } // for
} /* at91_eth_deliver_rx() */

// ----------------------------------------------------------------------------
// The delivery function is called from thread context, after the DSR has woken
// up the packet handling thread. It needs to report completed transmits so
// that higher-level code can release the tx pbuf if needed, and report all
// received frames.

__externC void
cyg_lwip_eth_run_deliveries(void)
{
    at91_eth *eth = &at91_eth_info;
    cyg_uint32 intstat, tsr, rsr;

    //    DIAG(1, "entry");

    for (;;)
    {
        // Read and thus clear the interrupt status. We don't use this to
        // determine cause - instead we use receive/transmit status directly.
        READ32(AT91_ETH_BASE(eth) + AT91_EMAC_ISR, intstat);

        /* Get the Transmit Status */
        READ32(AT91_ETH_BASE(eth)+AT91_EMAC_TSR, tsr);
        WRITE32(AT91_ETH_BASE(eth)+AT91_EMAC_TSR, tsr);
           
        /* Get the Receive Status */
        READ32(AT91_ETH_BASE(eth)+AT91_EMAC_RSR, rsr);
        WRITE32(AT91_ETH_BASE(eth)+AT91_EMAC_RSR, rsr);

        DIAG(1, "tsr 0x%08x rsr 0x%08x", tsr, rsr);

        // Abort if there are no events to process
        if( tsr == 0 && rsr == 0 )
            break;

        // Either successful frame transmission, or there was an error sending.
        // Either way, call tx delivery function.
        // No need for deliver_tx to also check tsr since we just have.
        if (tsr & at91_eth_tsr_tx_done_bits)
            at91_eth_deliver_tx(eth, false);

        // Successful frame reception
        if (rsr & AT91_EMAC_RSR_REC)
            at91_eth_deliver_rx(eth);

        // Don't care about RX BNA, nor TX UBR.
        // Other errors want handling, or if debugging, noticing.
        if (rsr & AT91_EMAC_RSR_OVR)
            DIAG(1, "Oops, detected receive overrun - ignoring");

        /* Even though we handled most TX issues, we want to spit out a debug message
         * for the odder ones.
         */
        if (tsr & AT91_EMAC_TSR_COL)
            DIAG(2, "Oops, detected tx collision. Handled.");
        if (tsr & AT91_EMAC_TSR_RLE)
            DIAG(1, "Oops, detected excessive tx retries. Handled.");
        if (tsr & AT91_EMAC_TSR_BEX)
            DIAG(1, "Oops, detected empty tx buffer mid-frame. Handled.");
        if (tsr & AT91_EMAC_TSR_UND)
            DIAG(1, "Oops, detected TX underrun. Handled.");

        /* Sanity check HRESP - should only get set for receive overruns
         * or transmit underruns, but I'm suspicious.
         */
        if (intstat & AT91_EMAC_ISR_HRESP)
        {
            if (0 == (rsr & AT91_EMAC_RSR_OVR) &&
                0 == (tsr & AT91_EMAC_TSR_UND))
            {
                DIAG(1, "HRESP set but no rx overrun, or tx underrun!!! Tell Jifl");
            }
        }

        /* Nothing else wants explicit handling as far as I'm concerned. */
    }
    
    // Allow more interrupts.
    cyg_drv_interrupt_unmask(AT91_ETH_INTR_VECTOR(eth));
}

// ----------------------------------------------------------------------------

// SET_MAC_ADDRESS is straightforward, it just requires updating two registers.

__externC int
cyg_lwip_eth_ioctl(struct netif *netif, unsigned long key, void* data, int data_length)
{
    at91_eth*    eth = (at91_eth*) netif->state;

    DIAG(1, "entry");
    
    switch(key) {
      case ETH_DRV_SET_MAC_ADDRESS:
        {
            memcpy(eth->mac, data, 6);
            at91_set_mac(eth, eth->mac, 1);
            return 0;
        }

      default:
        return 1;
    }
}

// ----------------------------------------------------------------------------
/* EOF if_at91-lwip.c */
