I’m afraid I have no interest in Arduino for ST parts, but I do use STM32 parts extensively. Sometimes I use the ST provided peripheral interface libraries, which for setting up a timer for encoder input works for me as follows:
/*
* Setup the GPIO
* PB4: ENCA
* PB5: ENCB
*/
GPIO_InitStructure.GPIO_OType = GPIO_OType_PP;
GPIO_InitStructure.GPIO_PuPd = GPIO_PuPd_NOPULL;
GPIO_InitStructure.GPIO_Speed = GPIO_Speed_Level_2;
GPIO_InitStructure.GPIO_Mode = GPIO_Mode_AF;
GPIO_InitStructure.GPIO_Pin = GPIO_Pin_4 | GPIO_Pin_5;
GPIO_Init(GPIOB, &GPIO_InitStructure);
/*
* Route the inputs to TIM3 for encoder operation
*/
GPIO_PinAFConfig(GPIOB, GPIO_PinSource4, GPIO_AF_2);
GPIO_PinAFConfig(GPIOB, GPIO_PinSource5, GPIO_AF_2);
/*
* Setup TIM3 to monitor the encoder
*/
RCC_APB1PeriphClockCmd(RCC_APB1Periph_TIM3, ENABLE);
TIM_InitStructure.TIM_Prescaler = 0;
TIM_InitStructure.TIM_CounterMode = TIM_CounterMode_Up;
TIM_InitStructure.TIM_Period = (-1);
TIM_InitStructure.TIM_ClockDivision = 0;
TIM_InitStructure.TIM_RepetitionCounter = 0;
TIM_TimeBaseInit(TIM3, &TIM_InitStructure);
TIM_ICInitStructure.TIM_Channel = TIM_Channel_1;
TIM_ICInitStructure.TIM_ICPolarity = TIM_ICPolarity_BothEdge;
TIM_ICInitStructure.TIM_ICSelection = TIM_ICSelection_DirectTI;
TIM_ICInitStructure.TIM_ICPrescaler = TIM_ICPSC_DIV1;
TIM_ICInitStructure.TIM_ICFilter = 7;
TIM_ICInit(TIM3, &TIM_ICInitStructure);
TIM_ICInitStructure.TIM_Channel = TIM_Channel_2;
TIM_ICInit(TIM3, &TIM_ICInitStructure);
TIM_EncoderInterfaceConfig(TIM3, TIM_EncoderMode_TI12, TIM_ICPolarity_BothEdge, TIM_ICPolarity_BothEdge);
TIM_OC1PolarityConfig(TIM3, TIM_OCPolarity_High);
TIM_OC1NPolarityConfig(TIM3, TIM_OCPolarity_High);
TIM_OC2PolarityConfig(TIM3, TIM_OCPolarity_High);
TIM_OC2NPolarityConfig(TIM3, TIM_OCPolarity_High);
TIM_Cmd(TIM3, ENABLE);