diff options
Diffstat (limited to 'drivers/ddr/marvell/axp/ddr3_read_leveling.c')
| -rw-r--r-- | drivers/ddr/marvell/axp/ddr3_read_leveling.c | 1214 |
1 files changed, 1214 insertions, 0 deletions
diff --git a/drivers/ddr/marvell/axp/ddr3_read_leveling.c b/drivers/ddr/marvell/axp/ddr3_read_leveling.c new file mode 100644 index 0000000000..4662bde994 --- /dev/null +++ b/drivers/ddr/marvell/axp/ddr3_read_leveling.c @@ -0,0 +1,1214 @@ +/* + * Copyright (C) Marvell International Ltd. and its affiliates + * + * SPDX-License-Identifier: GPL-2.0 + */ + +#include <common.h> +#include <i2c.h> +#include <spl.h> +#include <asm/io.h> +#include <asm/arch/cpu.h> +#include <asm/arch/soc.h> + +#include "ddr3_hw_training.h" + +/* + * Debug + */ +#define DEBUG_RL_C(s, d, l) \ + DEBUG_RL_S(s); DEBUG_RL_D(d, l); DEBUG_RL_S("\n") +#define DEBUG_RL_FULL_C(s, d, l) \ + DEBUG_RL_FULL_S(s); DEBUG_RL_FULL_D(d, l); DEBUG_RL_FULL_S("\n") + +#ifdef MV_DEBUG_RL +#define DEBUG_RL_S(s) \ + debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s) +#define DEBUG_RL_D(d, l) \ + debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d) +#else +#define DEBUG_RL_S(s) +#define DEBUG_RL_D(d, l) +#endif + +#ifdef MV_DEBUG_RL_FULL +#define DEBUG_RL_FULL_S(s) puts(s) +#define DEBUG_RL_FULL_D(d, l) printf("%x", d) +#else +#define DEBUG_RL_FULL_S(s) +#define DEBUG_RL_FULL_D(d, l) +#endif + +extern u32 rl_pattern[LEN_STD_PATTERN]; + +#ifdef RL_MODE +static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq, + int ratio_2to1, u32 ecc, + MV_DRAM_INFO *dram_info); +#else +static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq, + int ratio_2to1, u32 ecc, + MV_DRAM_INFO *dram_info); +#endif + +/* + * Name: ddr3_read_leveling_hw + * Desc: Execute the Read leveling phase by HW + * Args: dram_info - main struct + * freq - current sequence frequency + * Notes: + * Returns: MV_OK if success, MV_FAIL if fail. + */ +int ddr3_read_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info) +{ + u32 reg; + + /* Debug message - Start Read leveling procedure */ + DEBUG_RL_S("DDR3 - Read Leveling - Starting HW RL procedure\n"); + + /* Start Auto Read Leveling procedure */ + reg = 1 << REG_DRAM_TRAINING_RL_OFFS; + /* Config the retest number */ + reg |= (COUNT_HW_RL << REG_DRAM_TRAINING_RETEST_OFFS); + + /* Enable CS in the automatic process */ + reg |= (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS); + + reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */ + + reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) | + (1 << REG_DRAM_TRAINING_AUTO_OFFS); + reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg); + + /* Wait */ + do { + reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) & + (1 << REG_DRAM_TRAINING_AUTO_OFFS); + } while (reg); /* Wait for '0' */ + + /* Check if Successful */ + if (reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) & + (1 << REG_DRAM_TRAINING_ERROR_OFFS)) { + u32 delay, phase, pup, cs; + + dram_info->rl_max_phase = 0; + dram_info->rl_min_phase = 10; + + /* Read results to arrays */ + for (cs = 0; cs < MAX_CS; cs++) { + if (dram_info->cs_ena & (1 << cs)) { + for (pup = 0; + pup < dram_info->num_of_total_pups; + pup++) { + if (pup == dram_info->num_of_std_pups + && dram_info->ecc_ena) + pup = ECC_PUP; + reg = + ddr3_read_pup_reg(PUP_RL_MODE, cs, + pup); + phase = (reg >> REG_PHY_PHASE_OFFS) & + PUP_PHASE_MASK; + delay = reg & PUP_DELAY_MASK; + dram_info->rl_val[cs][pup][P] = phase; + if (phase > dram_info->rl_max_phase) + dram_info->rl_max_phase = phase; + if (phase < dram_info->rl_min_phase) + dram_info->rl_min_phase = phase; + dram_info->rl_val[cs][pup][D] = delay; + dram_info->rl_val[cs][pup][S] = + RL_FINAL_STATE; + reg = + ddr3_read_pup_reg(PUP_RL_MODE + 0x1, + cs, pup); + dram_info->rl_val[cs][pup][DQS] = + (reg & 0x3F); + } +#ifdef MV_DEBUG_RL + /* Print results */ + DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ", + (u32) cs, 1); + + for (pup = 0; + pup < (dram_info->num_of_total_pups); + pup++) { + if (pup == dram_info->num_of_std_pups + && dram_info->ecc_ena) + pup = ECC_PUP; + DEBUG_RL_S("DDR3 - Read Leveling - PUP: "); + DEBUG_RL_D((u32) pup, 1); + DEBUG_RL_S(", Phase: "); + DEBUG_RL_D((u32) dram_info-> + rl_val[cs][pup][P], 1); + DEBUG_RL_S(", Delay: "); + DEBUG_RL_D((u32) dram_info-> + rl_val[cs][pup][D], 2); + DEBUG_RL_S("\n"); + } +#endif + } + } + + dram_info->rd_rdy_dly = + reg_read(REG_READ_DATA_READY_DELAYS_ADDR) & + REG_READ_DATA_SAMPLE_DELAYS_MASK; + dram_info->rd_smpl_dly = + reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR) & + REG_READ_DATA_READY_DELAYS_MASK; +#ifdef MV_DEBUG_RL + DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ", + dram_info->rd_smpl_dly, 2); + DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ", + dram_info->rd_rdy_dly, 2); + DEBUG_RL_S("DDR3 - Read Leveling - HW RL Ended Successfully\n"); +#endif + return MV_OK; + + } else { + DEBUG_RL_S("DDR3 - Read Leveling - HW RL Error\n"); + return MV_FAIL; + } +} + +/* + * Name: ddr3_read_leveling_sw + * Desc: Execute the Read leveling phase by SW + * Args: dram_info - main struct + * freq - current sequence frequency + * Notes: + * Returns: MV_OK if success, MV_FAIL if fail. + */ +int ddr3_read_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info) +{ + u32 reg, cs, ecc, pup_num, phase, delay, pup; + int status; + + /* Debug message - Start Read leveling procedure */ + DEBUG_RL_S("DDR3 - Read Leveling - Starting SW RL procedure\n"); + + /* Enable SW Read Leveling */ + reg = reg_read(REG_DRAM_TRAINING_2_ADDR) | + (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS); + reg &= ~(1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS); + /* [0]=1 - Enable SW override */ + /* 0x15B8 - Training SW 2 Register */ + reg_write(REG_DRAM_TRAINING_2_ADDR, reg); + +#ifdef RL_MODE + reg = (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS) | + (1 << REG_DRAM_TRAINING_AUTO_OFFS); + reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */ +#endif + + /* Loop for each CS */ + for (cs = 0; cs < dram_info->num_cs; cs++) { + DEBUG_RL_C("DDR3 - Read Leveling - CS - ", (u32) cs, 1); + + for (ecc = 0; ecc <= (dram_info->ecc_ena); ecc++) { + /* ECC Support - Switch ECC Mux on ecc=1 */ + reg = reg_read(REG_DRAM_TRAINING_2_ADDR) & + ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS); + reg |= (dram_info->ecc_ena * + ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS); + reg_write(REG_DRAM_TRAINING_2_ADDR, reg); + + if (ecc) + DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Enabled\n"); + else + DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Disabled\n"); + + /* Set current sample delays */ + reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR); + reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK << + (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs)); + reg |= (dram_info->cl << + (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs)); + reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, reg); + + /* Set current Ready delay */ + reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR); + reg &= ~(REG_READ_DATA_READY_DELAYS_MASK << + (REG_READ_DATA_READY_DELAYS_OFFS * cs)); + if (!ratio_2to1) { + /* 1:1 mode */ + reg |= ((dram_info->cl + 1) << + (REG_READ_DATA_READY_DELAYS_OFFS * cs)); + } else { + /* 2:1 mode */ + reg |= ((dram_info->cl + 2) << + (REG_READ_DATA_READY_DELAYS_OFFS * cs)); + } + reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg); + + /* Read leveling Single CS[cs] */ +#ifdef RL_MODE + status = + ddr3_read_leveling_single_cs_rl_mode(cs, freq, + ratio_2to1, + ecc, + dram_info); + if (MV_OK != status) + return status; +#else + status = + ddr3_read_leveling_single_cs_window_mode(cs, freq, + ratio_2to1, + ecc, + dram_info) + if (MV_OK != status) + return status; +#endif + } + + /* Print results */ + DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ", (u32) cs, + 1); + + for (pup = 0; + pup < (dram_info->num_of_std_pups + dram_info->ecc_ena); + pup++) { + DEBUG_RL_S("DDR3 - Read Leveling - PUP: "); + DEBUG_RL_D((u32) pup, 1); + DEBUG_RL_S(", Phase: "); + DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][P], 1); + DEBUG_RL_S(", Delay: "); + DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][D], 2); + DEBUG_RL_S("\n"); + } + + DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ", + dram_info->rd_smpl_dly, 2); + DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ", + dram_info->rd_rdy_dly, 2); + + /* Configure PHY with average of 3 locked leveling settings */ + for (pup = 0; + pup < (dram_info->num_of_std_pups + dram_info->ecc_ena); + pup++) { + /* ECC support - bit 8 */ + pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup; + + /* For now, set last cnt result */ + phase = dram_info->rl_val[cs][pup][P]; + delay = dram_info->rl_val[cs][pup][D]; + ddr3_write_pup_reg(PUP_RL_MODE, cs, pup_num, phase, + delay); + } + } + + /* Reset PHY read FIFO */ + reg = reg_read(REG_DRAM_TRAINING_2_ADDR) | + (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); + /* 0x15B8 - Training SW 2 Register */ + reg_write(REG_DRAM_TRAINING_2_ADDR, reg); + + do { + reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) & + (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); + } while (reg); /* Wait for '0' */ + + /* ECC Support - Switch ECC Mux off ecc=0 */ + reg = reg_read(REG_DRAM_TRAINING_2_ADDR) & + ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS); + reg_write(REG_DRAM_TRAINING_2_ADDR, reg); + +#ifdef RL_MODE + reg_write(REG_DRAM_TRAINING_ADDR, 0); /* 0x15B0 - Training Register */ +#endif + + /* Disable SW Read Leveling */ + reg = reg_read(REG_DRAM_TRAINING_2_ADDR) & + ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS); + /* [0] = 0 - Disable SW override */ + reg = (reg | (0x1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS)); + /* [3] = 1 - Disable RL MODE */ + /* 0x15B8 - Training SW 2 Register */ + reg_write(REG_DRAM_TRAINING_2_ADDR, reg); + + DEBUG_RL_S("DDR3 - Read Leveling - Finished RL procedure for all CS\n"); + return MV_OK; +} + +#ifdef RL_MODE +/* + * overrun() extracted from ddr3_read_leveling_single_cs_rl_mode(). + * This just got too much indented making it hard to read / edit. + */ +static void overrun(u32 cs, MV_DRAM_INFO *info, u32 pup, u32 locked_pups, + u32 *locked_sum, u32 ecc, int *first_octet_locked, + int *counter_in_progress, int final_delay, u32 delay, + u32 phase) +{ + /* If no OverRun */ + if (((~locked_pups >> pup) & 0x1) && (final_delay == 0)) { + int idx; + + idx = pup + ecc * ECC_BIT; + + /* PUP passed, start examining */ + if (info->rl_val[cs][idx][S] == RL_UNLOCK_STATE) { + /* Must be RL_UNLOCK_STATE */ + /* Match expected value ? - Update State Machine */ + if (info->rl_val[cs][idx][C] < RL_RETRY_COUNT) { + DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ", + (u32)pup, 1); + info->rl_val[cs][idx][C]++; + + /* If pup got to last state - lock the delays */ + if (info->rl_val[cs][idx][C] == RL_RETRY_COUNT) { + info->rl_val[cs][idx][C] = 0; + info->rl_val[cs][idx][DS] = delay; + info->rl_val[cs][idx][PS] = phase; + + /* Go to Final State */ + info->rl_val[cs][idx][S] = RL_FINAL_STATE; + *locked_sum = *locked_sum + 1; + DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have locked pup: ", + (u32)pup, 1); + + /* + * If first lock - need to lock delays + */ + if (*first_octet_locked == 0) { + DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ", + (u32)pup, 1); + *first_octet_locked = 1; + } + + /* + * If pup is in not in final state but + * there was match - dont increment + * counter + */ + } else { + *counter_in_progress = 1; + } + } + } + } +} + +/* + * Name: ddr3_read_leveling_single_cs_rl_mode + * Desc: Execute Read leveling for single Chip select + * Args: cs - current chip select + * freq - current sequence frequency + * ecc - ecc iteration indication + * dram_info - main struct + * Notes: + * Returns: MV_OK if success, MV_FAIL if fail. + */ +static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq, + int ratio_2to1, u32 ecc, + MV_DRAM_INFO *dram_info) +{ + u32 reg, delay, phase, pup, rd_sample_delay, add, locked_pups, + repeat_max_cnt, sdram_offset, locked_sum; + u32 phase_min, ui_max_delay; + int all_locked, first_octet_locked, counter_in_progress; + int final_delay = 0; + + DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1); + + /* Init values */ + phase = 0; + delay = 0; + rd_sample_delay = dram_info->cl; + all_locked = 0; + first_octet_locked = 0; + repeat_max_cnt = 0; + locked_sum = 0; + + for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); + pup++) + dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0; + + /* Main loop */ + while (!all_locked) { + counter_in_progress = 0; + + DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = "); + DEBUG_RL_FULL_D(rd_sample_delay, 2); + DEBUG_RL_FULL_S(", RdRdyDly = "); + DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2); + DEBUG_RL_FULL_S(", Phase = "); + DEBUG_RL_FULL_D(phase, 1); + DEBUG_RL_FULL_S(", Delay = "); + DEBUG_RL_FULL_D(delay, 2); + DEBUG_RL_FULL_S("\n"); + + /* + * Broadcast to all PUPs current RL delays: DQS phase, + * leveling delay + */ + ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay); + + /* Reset PHY read FIFO */ + reg = reg_read(REG_DRAM_TRAINING_2_ADDR) | + (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); + /* 0x15B8 - Training SW 2 Register */ + reg_write(REG_DRAM_TRAINING_2_ADDR, reg); + + do { + reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) & + (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); + } while (reg); /* Wait for '0' */ + + /* Read pattern from SDRAM */ + sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS; + locked_pups = 0; + if (MV_OK != + ddr3_sdram_compare(dram_info, 0xFF, &locked_pups, + rl_pattern, LEN_STD_PATTERN, + sdram_offset, 0, 0, NULL, 0)) + return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PATTERN; + + /* Octet evaluation */ + /* pup_num = Q or 1 for ECC */ + for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) { + /* Check Overrun */ + if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >> + (REG_DRAM_TRAINING_2_OVERRUN_OFFS + pup)) & 0x1)) { + overrun(cs, dram_info, pup, locked_pups, + &locked_sum, ecc, &first_octet_locked, + &counter_in_progress, final_delay, + delay, phase); + } else { + DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ", + (u32)pup, 1); + } + } + + if (locked_sum == (dram_info->num_of_std_pups * + (1 - ecc) + ecc)) { + all_locked = 1; + DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n"); + } + + /* + * This is a fix for unstable condition where pups are + * toggling between match and no match + */ + /* + * If some of the pups is >1 <3, check if we did it too + * many times + */ + if (counter_in_progress == 1) { + /* Notify at least one Counter is >=1 and < 3 */ + if (repeat_max_cnt < RL_RETRY_COUNT) { + repeat_max_cnt++; + counter_in_progress = 1; + DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n"); + DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n"); + } else { + DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n"); + counter_in_progress = 0; + } + } + + /* + * Check some of the pups are in the middle of state machine + * and don't increment the delays + */ + if (!counter_in_progress && !all_locked) { + int idx; + + idx = pup + ecc * ECC_BIT; + + repeat_max_cnt = 0; + /* if 1:1 mode */ + if ((!ratio_2to1) && ((phase == 0) || (phase == 4))) + ui_max_delay = MAX_DELAY_INV; + else + ui_max_delay = MAX_DELAY; + + /* Increment Delay */ + if (delay < ui_max_delay) { + delay++; + /* + * Mark the last delay/pahse place for + * window final place + */ + if (delay == ui_max_delay) { + if ((!ratio_2to1 && phase == + MAX_PHASE_RL_L_1TO1) + || (ratio_2to1 && phase == + MAX_PHASE_RL_L_2TO1)) + final_delay = 1; + } + } else { + /* Phase+CL Incrementation */ + delay = 0; + + if (!ratio_2to1) { + /* 1:1 mode */ + if (first_octet_locked) { + /* some Pup was Locked */ + if (phase < MAX_PHASE_RL_L_1TO1) { + if (phase == 1) { + phase = 4; + } else { + phase++; + delay = MIN_DELAY_PHASE_1_LIMIT; + } + } else { + DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); + DEBUG_RL_S("1)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked n"); + return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK; + } + } else { + /* NO Pup was Locked */ + if (phase < MAX_PHASE_RL_UL_1TO1) { + phase++; + delay = + MIN_DELAY_PHASE_1_LIMIT; + } else { + phase = 0; + } + } + } else { + /* 2:1 mode */ + if (first_octet_locked) { + /* some Pup was Locked */ + if (phase < MAX_PHASE_RL_L_2TO1) { + phase++; + } else { + DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); + DEBUG_RL_S("2)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); + for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) { + /* pup_num = Q or 1 for ECC */ + if (dram_info->rl_val[cs][idx][S] + == 0) { + DEBUG_RL_C("Failed byte is = ", + pup, 1); + } + } + return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK; + } + } else { + /* No Pup was Locked */ + if (phase < MAX_PHASE_RL_UL_2TO1) + phase++; + else + phase = 0; + } + } + + /* + * If we finished a full Phases cycle (so now + * phase = 0, need to increment rd_sample_dly + */ + if (phase == 0 && first_octet_locked == 0) { + rd_sample_delay++; + if (rd_sample_delay == 0x10) { + DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); + DEBUG_RL_S("3)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); + for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) { + /* pup_num = Q or 1 for ECC */ + if (dram_info-> + rl_val[cs][idx][S] == 0) { + DEBUG_RL_C("Failed byte is = ", + pup, 1); + } + } + return MV_DDR3_TRAINING_ERR_RD_LVL_PUP_UNLOCK; + } + + /* Set current rd_sample_delay */ + reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR); + reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK + << (REG_READ_DATA_SAMPLE_DELAYS_OFFS + * cs)); + reg |= (rd_sample_delay << + (REG_READ_DATA_SAMPLE_DELAYS_OFFS * + cs)); + reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, + reg); + } + + /* + * Set current rdReadyDelay according to the + * hash table (Need to do this in every phase + * change) + */ + if (!ratio_2to1) { + /* 1:1 mode */ + add = reg_read(REG_TRAINING_DEBUG_2_ADDR); + switch (phase) { + case 0: + add = (add >> + REG_TRAINING_DEBUG_2_OFFS); + break; + case 1: + add = (add >> + (REG_TRAINING_DEBUG_2_OFFS + + 3)); + break; + case 4: + add = (add >> + (REG_TRAINING_DEBUG_2_OFFS + + 6)); + break; + case 5: + add = (add >> + (REG_TRAINING_DEBUG_2_OFFS + + 9)); + break; + } + add &= REG_TRAINING_DEBUG_2_MASK; + } else { + /* 2:1 mode */ + add = reg_read(REG_TRAINING_DEBUG_3_ADDR); + add = (add >> + (phase * + REG_TRAINING_DEBUG_3_OFFS)); + add &= REG_TRAINING_DEBUG_3_MASK; + } + + reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR); + reg &= ~(REG_READ_DATA_READY_DELAYS_MASK << + (REG_READ_DATA_READY_DELAYS_OFFS * cs)); + reg |= ((rd_sample_delay + add) << + (REG_READ_DATA_READY_DELAYS_OFFS * cs)); + reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg); + dram_info->rd_smpl_dly = rd_sample_delay; + dram_info->rd_rdy_dly = rd_sample_delay + add; + } + + /* Reset counters for pups with states<RD_STATE_COUNT */ + for (pup = 0; pup < + (dram_info->num_of_std_pups * (1 - ecc) + ecc); + pup++) { + if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT) + dram_info->rl_val[cs][idx][C] = 0; + } + } + } + + phase_min = 10; + + for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) { + if (dram_info->rl_val[cs][pup][PS] < phase_min) + phase_min = dram_info->rl_val[cs][pup][PS]; + } + + /* + * Set current rdReadyDelay according to the hash table (Need to + * do this in every phase change) + */ + if (!ratio_2to1) { + /* 1:1 mode */ + add = reg_read(REG_TRAINING_DEBUG_2_ADDR); + switch (phase_min) { + case 0: + add = (add >> REG_TRAINING_DEBUG_2_OFFS); + break; + case 1: + add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3)); + break; + case 4: + add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6)); + break; + case 5: + add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9)); + break; + } + add &= REG_TRAINING_DEBUG_2_MASK; + } else { + /* 2:1 mode */ + add = reg_read(REG_TRAINING_DEBUG_3_ADDR); + add = (add >> (phase_min * REG_TRAINING_DEBUG_3_OFFS)); + add &= REG_TRAINING_DEBUG_3_MASK; + } + + reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR); + reg &= ~(REG_READ_DATA_READY_DELAYS_MASK << + (REG_READ_DATA_READY_DELAYS_OFFS * cs)); + reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); + reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg); + dram_info->rd_rdy_dly = rd_sample_delay + add; + + for (cs = 0; cs < dram_info->num_cs; cs++) { + for (pup = 0; pup < dram_info->num_of_total_pups; pup++) { + reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup); + dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F); + } + } + + return MV_OK; +} + +#else + +/* + * Name: ddr3_read_leveling_single_cs_window_mode + * Desc: Execute Read leveling for single Chip select + * Args: cs - current chip select + * freq - current sequence frequency + * ecc - ecc iteration indication + * dram_info - main struct + * Notes: + * Returns: MV_OK if success, MV_FAIL if fail. + */ +static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq, + int ratio_2to1, u32 ecc, + MV_DRAM_INFO *dram_info) +{ + u32 reg, delay, phase, sum, pup, rd_sample_delay, add, locked_pups, + repeat_max_cnt, sdram_offset, final_sum, locked_sum; + u32 delay_s, delay_e, tmp, phase_min, ui_max_delay; + int all_locked, first_octet_locked, counter_in_progress; + int final_delay = 0; + + DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1); + + /* Init values */ + phase = 0; + delay = 0; + rd_sample_delay = dram_info->cl; + all_locked = 0; + first_octet_locked = 0; + repeat_max_cnt = 0; + sum = 0; + final_sum = 0; + locked_sum = 0; + + for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); + pup++) + dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0; + + /* Main loop */ + while (!all_locked) { + counter_in_progress = 0; + + DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = "); + DEBUG_RL_FULL_D(rd_sample_delay, 2); + DEBUG_RL_FULL_S(", RdRdyDly = "); + DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2); + DEBUG_RL_FULL_S(", Phase = "); + DEBUG_RL_FULL_D(phase, 1); + DEBUG_RL_FULL_S(", Delay = "); + DEBUG_RL_FULL_D(delay, 2); + DEBUG_RL_FULL_S("\n"); + + /* + * Broadcast to all PUPs current RL delays: DQS phase,leveling + * delay + */ + ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay); + + /* Reset PHY read FIFO */ + reg = reg_read(REG_DRAM_TRAINING_2_ADDR) | + (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); + /* 0x15B8 - Training SW 2 Register */ + reg_write(REG_DRAM_TRAINING_2_ADDR, reg); + + do { + reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) & + (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); + } while (reg); /* Wait for '0' */ + + /* Read pattern from SDRAM */ + sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS; + locked_pups = 0; + if (MV_OK != + ddr3_sdram_compare(dram_info, 0xFF, &locked_pups, + rl_pattern, LEN_STD_PATTERN, + sdram_offset, 0, 0, NULL, 0)) + return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PATTERN; + + /* Octet evaluation */ + for (pup = 0; pup < (dram_info->num_of_std_pups * + (1 - ecc) + ecc); pup++) { + /* pup_num = Q or 1 for ECC */ + int idx; + + idx = pup + ecc * ECC_BIT; + + /* Check Overrun */ + if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >> + (REG_DRAM_TRAINING_2_OVERRUN_OFFS + + pup)) & 0x1)) { + /* If no OverRun */ + + /* Inside the window */ + if (dram_info->rl_val[cs][idx][S] == RL_WINDOW_STATE) { + /* + * Match expected value ? - Update + * State Machine + */ + if (((~locked_pups >> pup) & 0x1) + && (final_delay == 0)) { + /* Match - Still inside the Window */ + DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got another match inside the window for pup: ", + (u32)pup, 1); + + } else { + /* We got fail -> this is the end of the window */ + dram_info->rl_val[cs][idx][DE] = delay; + dram_info->rl_val[cs][idx][PE] = phase; + /* Go to Final State */ + dram_info->rl_val[cs][idx][S]++; + final_sum++; + DEBUG_RL_FULL_C("DDR3 - Read Leveling - We finished the window for pup: ", + (u32)pup, 1); + } + + /* Before the start of the window */ + } else if (dram_info->rl_val[cs][idx][S] == + RL_UNLOCK_STATE) { + /* Must be RL_UNLOCK_STATE */ + /* + * Match expected value ? - Update + * State Machine + */ + if (dram_info->rl_val[cs][idx][C] < + RL_RETRY_COUNT) { + if (((~locked_pups >> pup) & 0x1)) { + /* Match */ + DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ", + (u32)pup, 1); + dram_info->rl_val[cs][idx][C]++; + + /* If pup got to last state - lock the delays */ + if (dram_info->rl_val[cs][idx][C] == + RL_RETRY_COUNT) { + dram_info->rl_val[cs][idx][C] = 0; + dram_info->rl_val[cs][idx][DS] = + delay; + dram_info->rl_val[cs][idx][PS] = + phase; + dram_info->rl_val[cs][idx][S]++; /* Go to Window State */ + locked_sum++; + /* Will count the pups that got locked */ + + /* IF First lock - need to lock delays */ + if (first_octet_locked == 0) { + DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ", + (u32)pup, 1); + first_octet_locked + = + 1; + } + } + + /* if pup is in not in final state but there was match - dont increment counter */ + else { + counter_in_progress + = 1; + } + } + } + } + } else { + DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ", + (u32)pup, 1); + counter_in_progress = 1; + } + } + + if (final_sum == (dram_info->num_of_std_pups * (1 - ecc) + ecc)) { + all_locked = 1; + DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n"); + } + + /* + * This is a fix for unstable condition where pups are + * toggling between match and no match + */ + /* + * If some of the pups is >1 <3, check if we did it too many + * times + */ + if (counter_in_progress == 1) { + if (repeat_max_cnt < RL_RETRY_COUNT) { + /* Notify at least one Counter is >=1 and < 3 */ + repeat_max_cnt++; + counter_in_progress = 1; + DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n"); + DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n"); + } else { + DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n"); + counter_in_progress = 0; + } + } + + /* + * Check some of the pups are in the middle of state machine + * and don't increment the delays + */ + if (!counter_in_progress && !all_locked) { + repeat_max_cnt = 0; + if (!ratio_2to1) + ui_max_delay = MAX_DELAY_INV; + else + ui_max_delay = MAX_DELAY; + + /* Increment Delay */ + if (delay < ui_max_delay) { + /* Delay Incrementation */ + delay++; + if (delay == ui_max_delay) { + /* + * Mark the last delay/pahse place + * for window final place + */ + if ((!ratio_2to1 + && phase == MAX_PHASE_RL_L_1TO1) + || (ratio_2to1 + && phase == + MAX_PHASE_RL_L_2TO1)) + final_delay = 1; + } + } else { + /* Phase+CL Incrementation */ + delay = 0; + if (!ratio_2to1) { + /* 1:1 mode */ + if (first_octet_locked) { + /* some pupet was Locked */ + if (phase < MAX_PHASE_RL_L_1TO1) { +#ifdef RL_WINDOW_WA + if (phase == 0) +#else + if (phase == 1) +#endif + phase = 4; + else + phase++; + } else { + DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); + return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK; + } + } else { + /* No Pup was Locked */ + if (phase < MAX_PHASE_RL_UL_1TO1) { +#ifdef RL_WINDOW_WA + if (phase == 0) + phase = 4; +#else + phase++; +#endif + } else + phase = 0; + } + } else { + /* 2:1 mode */ + if (first_octet_locked) { + /* Some Pup was Locked */ + if (phase < MAX_PHASE_RL_L_2TO1) { + phase++; + } else { + DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); + return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK; + } + } else { + /* No Pup was Locked */ + if (phase < MAX_PHASE_RL_UL_2TO1) + phase++; + else + phase = 0; + } + } + + /* + * If we finished a full Phases cycle (so + * now phase = 0, need to increment + * rd_sample_dly + */ + if (phase == 0 && first_octet_locked == 0) { + rd_sample_delay++; + + /* Set current rd_sample_delay */ + reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR); + reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK << + (REG_READ_DATA_SAMPLE_DELAYS_OFFS + * cs)); + reg |= (rd_sample_delay << + (REG_READ_DATA_SAMPLE_DELAYS_OFFS * + cs)); + reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, + reg); + } + + /* + * Set current rdReadyDelay according to the + * hash table (Need to do this in every phase + * change) + */ + if (!ratio_2to1) { + /* 1:1 mode */ + add = reg_read(REG_TRAINING_DEBUG_2_ADDR); + switch (phase) { + case 0: + add = add >> + REG_TRAINING_DEBUG_2_OFFS; + break; + case 1: + add = add >> + (REG_TRAINING_DEBUG_2_OFFS + + 3); + break; + case 4: + add = add >> + (REG_TRAINING_DEBUG_2_OFFS + + 6); + break; + case 5: + add = add >> + (REG_TRAINING_DEBUG_2_OFFS + + 9); + break; + } + } else { + /* 2:1 mode */ + add = reg_read(REG_TRAINING_DEBUG_3_ADDR); + add = (add >> phase * + REG_TRAINING_DEBUG_3_OFFS); + } + add &= REG_TRAINING_DEBUG_2_MASK; + reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR); + reg &= ~(REG_READ_DATA_READY_DELAYS_MASK << + (REG_READ_DATA_READY_DELAYS_OFFS * cs)); + reg |= ((rd_sample_delay + add) << + (REG_READ_DATA_READY_DELAYS_OFFS * cs)); + reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg); + dram_info->rd_smpl_dly = rd_sample_delay; + dram_info->rd_rdy_dly = rd_sample_delay + add; + } + + /* Reset counters for pups with states<RD_STATE_COUNT */ + for (pup = 0; + pup < + (dram_info->num_of_std_pups * (1 - ecc) + ecc); + pup++) { + if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT) + dram_info->rl_val[cs][idx][C] = 0; + } + } + } + + phase_min = 10; + + for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) { + DEBUG_RL_S("DDR3 - Read Leveling - Window info - PUP: "); + DEBUG_RL_D((u32) pup, 1); + DEBUG_RL_S(", PS: "); + DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PS], 1); + DEBUG_RL_S(", DS: "); + DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DS], 2); + DEBUG_RL_S(", PE: "); + DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PE], 1); + DEBUG_RL_S(", DE: "); + DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DE], 2); + DEBUG_RL_S("\n"); + } + + /* Find center of the window procedure */ + for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); + pup++) { +#ifdef RL_WINDOW_WA + if (!ratio_2to1) { /* 1:1 mode */ + if (dram_info->rl_val[cs][idx][PS] == 4) + dram_info->rl_val[cs][idx][PS] = 1; + if (dram_info->rl_val[cs][idx][PE] == 4) + dram_info->rl_val[cs][idx][PE] = 1; + + delay_s = dram_info->rl_val[cs][idx][PS] * + MAX_DELAY_INV + dram_info->rl_val[cs][idx][DS]; + delay_e = dram_info->rl_val[cs][idx][PE] * + MAX_DELAY_INV + dram_info->rl_val[cs][idx][DE]; + + tmp = (delay_e - delay_s) / 2 + delay_s; + phase = tmp / MAX_DELAY_INV; + if (phase == 1) /* 1:1 mode */ + phase = 4; + + if (phase < phase_min) /* for the read ready delay */ + phase_min = phase; + + dram_info->rl_val[cs][idx][P] = phase; + dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY_INV; + + } else { + delay_s = dram_info->rl_val[cs][idx][PS] * + MAX_DELAY + dram_info->rl_val[cs][idx][DS]; + delay_e = dram_info->rl_val[cs][idx][PE] * + MAX_DELAY + dram_info->rl_val[cs][idx][DE]; + + tmp = (delay_e - delay_s) / 2 + delay_s; + phase = tmp / MAX_DELAY; + + if (phase < phase_min) /* for the read ready delay */ + phase_min = phase; + + dram_info->rl_val[cs][idx][P] = phase; + dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY; + } +#else + if (!ratio_2to1) { /* 1:1 mode */ + if (dram_info->rl_val[cs][idx][PS] > 1) + dram_info->rl_val[cs][idx][PS] -= 2; + if (dram_info->rl_val[cs][idx][PE] > 1) + dram_info->rl_val[cs][idx][PE] -= 2; + } + + delay_s = dram_info->rl_val[cs][idx][PS] * MAX_DELAY + + dram_info->rl_val[cs][idx][DS]; + delay_e = dram_info->rl_val[cs][idx][PE] * MAX_DELAY + + dram_info->rl_val[cs][idx][DE]; + + tmp = (delay_e - delay_s) / 2 + delay_s; + phase = tmp / MAX_DELAY; + if (!ratio_2to1 && phase > 1) /* 1:1 mode */ + phase += 2; + + if (phase < phase_min) /* for the read ready delay */ + phase_min = phase; + + dram_info->rl_val[cs][idx][P] = phase; + dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY; +#endif + } + + /* Set current rdReadyDelay according to the hash table (Need to do this in every phase change) */ + if (!ratio_2to1) { /* 1:1 mode */ + add = reg_read(REG_TRAINING_DEBUG_2_ADDR); + switch (phase_min) { + case 0: + add = (add >> REG_TRAINING_DEBUG_2_OFFS); + break; + case 1: + add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3)); + break; + case 4: + add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6)); + break; + case 5: + add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9)); + break; + } + } else { /* 2:1 mode */ + add = reg_read(REG_TRAINING_DEBUG_3_ADDR); + add = (add >> phase_min * REG_TRAINING_DEBUG_3_OFFS); + } + + add &= REG_TRAINING_DEBUG_2_MASK; + reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR); + reg &= + ~(REG_READ_DATA_READY_DELAYS_MASK << + (REG_READ_DATA_READY_DELAYS_OFFS * cs)); + reg |= + ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); + reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg); + dram_info->rd_rdy_dly = rd_sample_delay + add; + + for (cs = 0; cs < dram_info->num_cs; cs++) { + for (pup = 0; pup < dram_info->num_of_total_pups; pup++) { + reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup); + dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F); + } + } + + return MV_OK; +} +#endif |